You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by aj...@apache.org on 2019/12/24 07:47:26 UTC
[carbondata] branch master updated:
[CARBONDATA-3609][CARBONDATA-3610] Remove preaggregate and timeseries
datamap
This is an automated email from the ASF dual-hosted git repository.
ajantha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 4a603d6 [CARBONDATA-3609][CARBONDATA-3610] Remove preaggregate and timeseries datamap
4a603d6 is described below
commit 4a603d6ada544fad67b9ea14d2b7d81cb1b03e83
Author: Jacky Li <ja...@qq.com>
AuthorDate: Sat Dec 21 21:47:41 2019 +0800
[CARBONDATA-3609][CARBONDATA-3610] Remove preaggregate and timeseries datamap
Remove preaggregate and timeseries datamap as the same features are
handled in MV.
This closes #3522
---
README.md | 2 -
.../core/constants/CarbonCommonConstants.java | 9 +-
.../constants/CarbonCommonConstantsInternal.java | 2 -
.../carbondata/core/datamap/DataMapProvider.java | 2 -
.../core/datamap/DataMapStoreManager.java | 5 -
.../carbondata/core/datamap/DataMapUtil.java | 2 +-
.../carbondata/core/metadata/SegmentFileStore.java | 2 +-
.../schema/datamap/DataMapClassProvider.java | 8 +-
.../schema/table/AggregationDataMapSchema.java | 385 ----
.../core/metadata/schema/table/CarbonTable.java | 22 +-
.../core/metadata/schema/table/DataMapSchema.java | 4 +-
.../schema/table/DataMapSchemaFactory.java | 12 +-
.../carbondata/core/preagg/AggregateQueryPlan.java | 49 -
.../core/preagg/AggregateTableSelector.java | 140 --
.../apache/carbondata/core/preagg/QueryColumn.java | 87 -
.../apache/carbondata/core/util/CarbonUtil.java | 52 +-
.../apache/carbondata/core/util/SessionParams.java | 5 -
.../datamap/bloom/BloomCoarseGrainDataMap.java | 21 +-
.../bloom/BloomCoarseGrainDataMapFactory.java | 2 +-
.../datamap/lucene/LuceneDataMapFactoryBase.java | 2 +-
.../carbondata/mv/datamap/MVAnalyzerRule.scala | 10 +-
.../carbondata/mv/datamap/MVDataMapProvider.scala | 14 +-
.../apache/carbondata/mv/datamap/MVHelper.scala | 11 +-
.../mv/rewrite/SummaryDatasetCatalog.scala | 3 +-
.../carbondata/mv/rewrite/MVCreateTestCase.scala | 6 +-
.../mv/rewrite/TestAllOperationsOnMV.scala | 42 +-
docs/datamap-developer-guide.md | 4 +-
docs/datamap/datamap-management.md | 13 +-
docs/datamap/preaggregate-datamap-guide.md | 275 ---
docs/datamap/timeseries-datamap-guide.md | 154 --
docs/introduction.md | 10 +-
docs/language-manual.md | 2 -
docs/presto-guide.md | 2 +-
docs/streaming-guide.md | 4 +-
docs/usecases.md | 3 +-
.../carbondata/examples/MVDataMapExample.scala | 3 +-
.../examples/PreAggregateDataMapExample.scala | 239 ---
.../TimeSeriesPreAggregateTableExample.scala | 108 --
.../apache/carbondata/examplesCI/RunExamples.scala | 8 -
format/src/main/thrift/schema.thrift | 5 +-
.../hadoop/api/CarbonFileInputFormat.java | 159 +-
.../carbondata/hadoop/api/CarbonInputFormat.java | 37 -
.../hadoop/api/CarbonOutputCommitter.java | 13 +-
.../hadoop/api/CarbonTableInputFormat.java | 95 +-
.../hadoop/util/CarbonInputFormatUtil.java | 22 +-
.../PartitionWithPreAggregateTestCase.scala | 255 ---
.../sdv/generated/PreAggregateTestCase.scala | 230 ---
.../generated/TimeSeriesPreAggregateTestCase.scala | 192 --
.../sdv/register/TestRegisterCarbonTable.scala | 67 -
.../carbondata/cluster/sdv/suite/SDVSuites.scala | 6 -
.../lucene/LuceneFineGrainDataMapSuite.scala | 18 -
.../complexType/TestComplexDataType.scala | 17 -
.../preaggregate/TestPreAggCreateCommand.scala | 534 ------
.../preaggregate/TestPreAggStreaming.scala | 143 --
.../preaggregate/TestPreAggregateCompaction.scala | 212 ---
.../preaggregate/TestPreAggregateDrop.scala | 169 --
.../preaggregate/TestPreAggregateExpressions.scala | 182 --
.../preaggregate/TestPreAggregateLoad.scala | 1039 -----------
.../preaggregate/TestPreAggregateMisc.scala | 81 -
.../TestPreAggregateTableSelection.scala | 466 -----
.../TestPreAggregateWithSubQuery.scala | 102 --
.../TestAdaptiveEncodingForPrimitiveTypes.scala | 16 -
.../timeseries/TestTimeSeriesCreateTable.scala | 604 ------
.../timeseries/TestTimeSeriesDropSuite.scala | 231 ---
.../TestTimeSeriesMatchStrategySuite.scala | 401 ----
.../TestTimeSeriesUnsupportedSuite.scala | 265 ---
.../timeseries/TestTimeseriesCompaction.scala | 136 --
.../timeseries/TestTimeseriesDataLoad.scala | 597 ------
.../timeseries/TestTimeseriesTableSelection.scala | 882 ---------
.../TestAlterTableSortColumnsProperty.scala | 32 -
...leWithColumnMetCacheAndCacheLevelProperty.scala | 19 -
.../compaction/TestHybridCompaction.scala | 27 -
.../createTable/TestRenameTableWithDataMap.scala | 48 -
.../CarbonIndexFileMergeTestCase.scala | 18 -
.../testsuite/datamap/TestDataMapCommand.scala | 368 ----
.../FlatFolderTableLoadingTestCase.scala | 20 -
.../testsuite/iud/DeleteCarbonTableTestCase.scala | 39 -
.../testsuite/iud/UpdateCarbonTableTestCase.scala | 40 -
.../longstring/VarcharDataTypesBasicTestCase.scala | 60 -
.../StandardPartitionTableQueryTestCase.scala | 15 -
...StandardPartitionWithPreaggregateTestCase.scala | 677 -------
.../carbondata/spark/util/SparkQueryTest.scala | 50 -
.../sql/commands/TestCarbonDropCacheCommand.scala | 66 -
.../sql/commands/TestCarbonShowCacheCommand.scala | 11 -
.../carbondata/events/CreateTableEvents.scala | 2 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 48 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 15 +-
.../org/apache/spark/sql/hive/CarbonAnalyzer.scala | 5 +-
.../apache/carbondata/datamap/DataMapManager.java | 8 +-
.../datamap/PreAggregateDataMapProvider.java | 120 --
.../datamap/TimeseriesDataMapProvider.java | 56 -
.../datamap/IndexDataMapRebuildRDD.scala | 2 +-
.../spark/rdd/AggregateDataMapCompactor.scala | 167 --
.../spark/rdd/CarbonDataRDDFactory.scala | 6 +-
.../carbondata/spark/rdd/CompactionFactory.scala | 26 +-
.../carbondata/spark/util/CarbonSparkUtil.scala | 6 +-
.../spark/sql/CarbonDatasourceHadoopRelation.scala | 12 -
.../scala/org/apache/spark/sql/CarbonEnv.scala | 40 +-
.../spark/sql/events/MergeIndexEventListener.scala | 5 +-
.../sql/execution/command/cache/CacheUtil.scala | 5 +-
.../command/cache/CarbonShowCacheCommand.scala | 2 +-
.../datamap/CarbonCreateDataMapCommand.scala | 15 +-
.../datamap/CarbonDataMapRebuildCommand.scala | 5 +-
.../command/datamap/CarbonDataMapShowCommand.scala | 85 +-
.../command/datamap/CarbonDropDataMapCommand.scala | 45 +-
.../command/management/CarbonAddLoadCommand.scala | 4 +-
.../CarbonAlterTableCompactionCommand.scala | 13 -
.../management/CarbonCleanFilesCommand.scala | 15 +-
.../management/CarbonInsertFromStageCommand.scala | 4 +-
.../command/management/CarbonLoadDataCommand.scala | 27 +-
.../command/mutation/HorizontalCompaction.scala | 6 +-
.../CarbonAlterTableAddHivePartitionCommand.scala | 2 +-
.../CarbonAlterTableDropHivePartitionCommand.scala | 22 +-
.../preaaggregate/PreAggregateListeners.scala | 932 ----------
.../preaaggregate/PreAggregateTableHelper.scala | 232 ---
.../command/preaaggregate/PreAggregateUtil.scala | 905 ---------
.../schema/CarbonAlterTableRenameCommand.scala | 2 +-
.../table/CarbonCreateTableLikeCommand.scala | 2 +-
.../command/table/CarbonDropTableCommand.scala | 33 +-
.../command/timeseries/TimeSeriesFunction.scala | 5 +-
.../command/timeseries/TimeSeriesUtil.scala | 15 +-
.../spark/sql/execution/strategy/DDLStrategy.scala | 2 +-
.../spark/sql/hive/CarbonAnalysisRules.scala | 13 +-
.../spark/sql/hive/CarbonFileMetastore.scala | 23 +-
.../spark/sql/hive/CarbonHiveMetaStore.scala | 15 -
.../apache/spark/sql/hive/CarbonMetaStore.scala | 3 -
.../spark/sql/hive/CarbonPreAggregateRules.scala | 1915 --------------------
.../org/apache/spark/sql/hive/CarbonRelation.scala | 2 +-
.../execution/command/CarbonHiveCommands.scala | 4 -
.../DropCacheEventListeners.scala | 10 +-
.../MVListeners.scala} | 30 +-
.../PrePrimingListener.scala | 7 +-
.../ShowCacheEventListeners.scala | 6 +-
.../apache/spark/sql/optimizer/CarbonFilters.scala | 39 -
.../spark/sql/optimizer/CarbonLateDecodeRule.scala | 46 +-
.../spark/sql/parser/CarbonSpark2SqlParser.scala | 12 +-
.../sql/parser/CarbonSparkSqlParserUtil.scala | 1 -
.../org/apache/spark/util/AlterTableUtil.scala | 18 +-
.../scala/org/apache/spark/util/DataMapUtil.scala | 2 +-
.../org/apache/spark/util/MergeIndexUtil.scala | 6 +-
.../apache/spark/sql/hive/CarbonSessionState.scala | 4 +-
.../BloomCoarseGrainDataMapFunctionSuite.scala | 97 -
.../carbondata/TestStreamingTableOpName.scala | 373 ----
.../register/TestRegisterCarbonTable.scala | 64 -
.../restructure/AlterTableValidationTestCase.scala | 18 -
.../vectorreader/AddColumnTestCases.scala | 14 -
.../AlterTableColumnRenameTestCase.scala | 16 -
.../vectorreader/ChangeDataTypeTestCases.scala | 17 -
.../vectorreader/DropColumnTestCases.scala | 16 -
.../org/apache/spark/util/CarbonCommandSuite.scala | 15 -
.../processing/loading/events/LoadEvents.java | 4 +-
.../processing/merger/CarbonDataMergerUtil.java | 12 +-
.../processing/util/CarbonLoaderUtil.java | 26 +-
153 files changed, 310 insertions(+), 15804 deletions(-)
diff --git a/README.md b/README.md
index da5b547..a34784d 100644
--- a/README.md
+++ b/README.md
@@ -58,8 +58,6 @@ CarbonData is built using Apache Maven, to [build CarbonData](https://github.com
* [CarbonData DataMap Management](https://github.com/apache/carbondata/blob/master/docs/datamap/datamap-management.md)
* [CarbonData BloomFilter DataMap](https://github.com/apache/carbondata/blob/master/docs/datamap/bloomfilter-datamap-guide.md)
* [CarbonData Lucene DataMap](https://github.com/apache/carbondata/blob/master/docs/datamap/lucene-datamap-guide.md)
- * [CarbonData Pre-aggregate DataMap](https://github.com/apache/carbondata/blob/master/docs/datamap/preaggregate-datamap-guide.md)
- * [CarbonData Timeseries DataMap](https://github.com/apache/carbondata/blob/master/docs/datamap/timeseries-datamap-guide.md)
* [CarbonData MV DataMap](https://github.com/apache/carbondata/blob/master/docs/datamap/mv-datamap-guide.md)
* [SDK Guide](https://github.com/apache/carbondata/blob/master/docs/sdk-guide.md)
* [C++ SDK Guide](https://github.com/apache/carbondata/blob/master/docs/csdk-guide.md)
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 2e3e7d3..1424e1a 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1513,15 +1513,8 @@ public final class CarbonCommonConstants {
public static final String CARBON_DATAMAP_VISIBLE = "carbon.datamap.visible.";
/**
- * Fetch and validate the segments.
- * Used for aggregate table load as segment validation is not required.
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String VALIDATE_CARBON_INPUT_SEGMENTS = "validate.carbon.input.segments.";
-
- /**
* Whether load/insert command is fired internally or by the user.
- * Used to block load/insert on pre-aggregate if fired by user
+ * Used to block load/insert on MV if fired by user
*/
@CarbonProperty
public static final String IS_INTERNAL_LOAD_CALL = "is.internal.load.call";
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstantsInternal.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstantsInternal.java
index 5111d25..3ea6494 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstantsInternal.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstantsInternal.java
@@ -23,8 +23,6 @@ package org.apache.carbondata.core.constants;
*/
public interface CarbonCommonConstantsInternal {
- String QUERY_ON_PRE_AGG_STREAMING = "carbon.query.on.preagg.streaming.";
-
String ROW_COUNT = "rowCount";
}
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java
index e58b61b..107e63f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java
@@ -64,8 +64,6 @@ import org.apache.log4j.Logger;
*
* <br>Currently CarbonData supports following provider:
* <ol>
- * <li> preaggregate: pre-aggregate table of single table </li>
- * <li> timeseries: pre-aggregate table based on time dimension of the table </li>
* <li> lucene: index backed by Apache Lucene </li>
* <li> bloomfilter: index backed by Bloom Filter </li>
* </ol>
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index 78c3290..c01a1ad 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -52,7 +52,6 @@ import org.apache.carbondata.core.util.CarbonSessionInfo;
import org.apache.carbondata.core.util.ThreadLocalSessionInfo;
import static org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.MV;
-import static org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.PREAGGREGATE;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
@@ -200,10 +199,6 @@ public final class DataMapStoreManager {
String dataBaseName = relationIdentifier.getDatabaseName();
String tableId = relationIdentifier.getTableId();
String providerName = dataMapSchema.getProviderName();
- // if the preaggregate datamap,not be modified the schema
- if (providerName.equalsIgnoreCase(PREAGGREGATE.toString())) {
- continue;
- }
// if the mv datamap,not be modified the relationIdentifier
if (!providerName.equalsIgnoreCase(MV.toString())) {
RelationIdentifier newRelationIdentifier = new RelationIdentifier(dataBaseName,
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapUtil.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapUtil.java
index 1a1200c..ca56962 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapUtil.java
@@ -261,7 +261,7 @@ public class DataMapUtil {
CarbonTable carbonTable, Configuration configuration) throws IOException {
SegmentStatusManager ssm =
new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier(), configuration);
- return ssm.getValidAndInvalidSegments(carbonTable.isChildTable());
+ return ssm.getValidAndInvalidSegments(carbonTable.isChildTableForMV());
}
/**
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
index caf1328..87b68c0 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
@@ -915,7 +915,7 @@ public class SegmentFileStore {
if (toBeDeleteSegments.size() > 0 || toBeUpdatedSegments.size() > 0) {
Set<Segment> segmentSet = new HashSet<>(
new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier())
- .getValidAndInvalidSegments(carbonTable.isChildTable()).getValidSegments());
+ .getValidAndInvalidSegments(carbonTable.isChildTableForMV()).getValidSegments());
CarbonUpdateUtil.updateTableMetadataStatus(segmentSet, carbonTable, uniqueId, true,
Segment.toSegmentList(toBeDeleteSegments, null),
Segment.toSegmentList(toBeUpdatedSegments, null), uuid);
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapClassProvider.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapClassProvider.java
index 33cf25a..4095857 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapClassProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapClassProvider.java
@@ -27,8 +27,6 @@ package org.apache.carbondata.core.metadata.schema.datamap;
*/
public enum DataMapClassProvider {
- PREAGGREGATE("org.apache.carbondata.core.datamap.AggregateDataMap", "preaggregate"),
- TIMESERIES("org.apache.carbondata.core.datamap.TimeSeriesDataMap", "timeseries"),
LUCENE("org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory", "lucene"),
BLOOMFILTER("org.apache.carbondata.datamap.bloom.BloomCoarseGrainDataMapFactory", "bloomfilter"),
MV("org.apache.carbondata.core.datamap.MVDataMap", "mv");
@@ -63,11 +61,7 @@ public enum DataMapClassProvider {
}
public static DataMapClassProvider getDataMapProviderOnName(String dataMapShortname) {
- if (TIMESERIES.isEqual(dataMapShortname)) {
- return TIMESERIES;
- } else if (PREAGGREGATE.isEqual(dataMapShortname)) {
- return PREAGGREGATE;
- } else if (LUCENE.isEqual(dataMapShortname)) {
+ if (LUCENE.isEqual(dataMapShortname)) {
return LUCENE;
} else if (BLOOMFILTER.isEqual(dataMapShortname)) {
return BLOOMFILTER;
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java
deleted file mode 100644
index 20174c5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.metadata.schema.table;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation;
-import org.apache.carbondata.core.preagg.TimeSeriesFunctionEnum;
-
-/**
- * data map schema class for pre aggregation
- */
-public class AggregationDataMapSchema extends DataMapSchema {
-
- private static final long serialVersionUID = 5900935117929888412L;
- /**
- * map of parent column name to set of child column column without
- * aggregation function
- */
- private Map<String, Set<ColumnSchema>> parentToNonAggChildMapping;
-
- /**
- * map of parent column name to set of child columns column with
- * aggregation function
- */
- private Map<String, Set<ColumnSchema>> parentToAggChildMapping;
-
- /**
- * map of parent column name to set of aggregation function applied in
- * in parent column
- */
- private Map<String, Set<String>> parentColumnToAggregationsMapping;
-
- /**
- * whether its a timeseries data map
- */
- private boolean isTimeseriesDataMap;
-
- /**
- * below ordinal will be used during sorting the data map
- * to support rollup for loading
- */
- private int ordinal = Integer.MAX_VALUE;
-
- // Dont remove transient otherwise serialization for carbonTable will fail using
- // JavaSerialization in spark.
- private transient Set aggExpToColumnMapping;
-
- AggregationDataMapSchema(String dataMapName, String className) {
- super(dataMapName, className);
- }
-
- public void setChildSchema(TableSchema childSchema) {
- super.setChildSchema(childSchema);
- List<ColumnSchema> listOfColumns = getChildSchema().getListOfColumns();
- fillNonAggFunctionColumns(listOfColumns);
- fillAggFunctionColumns(listOfColumns);
- fillParentNameToAggregationMapping(listOfColumns);
- }
-
- /**
- * Below method will be used to get the columns on which aggregate function
- * and time series function is not applied
- * @param columnName
- * parent column name
- * @return child column schema
- */
- public ColumnSchema getNonAggNonTimeseriesChildColBasedByParent(String columnName) {
- Set<ColumnSchema> columnSchemas = parentToNonAggChildMapping.get(columnName);
- if (null != columnSchemas) {
- Iterator<ColumnSchema> iterator = columnSchemas.iterator();
- while (iterator.hasNext()) {
- ColumnSchema next = iterator.next();
- if ((null == next.getAggFunction() || next.getAggFunction().isEmpty()) && null == next
- .getTimeSeriesFunction() || next.getTimeSeriesFunction().isEmpty()) {
- return next;
- }
- }
- }
- return null;
- }
-
- /**
- * Below method will be used to get the columns on which aggregate function is not applied
- * @param columnName
- * parent column name
- * @return child column schema
- */
- public ColumnSchema getNonAggChildColBasedByParent(String columnName) {
- Set<ColumnSchema> columnSchemas = parentToNonAggChildMapping.get(columnName);
- if (null != columnSchemas) {
- Iterator<ColumnSchema> iterator = columnSchemas.iterator();
- while (iterator.hasNext()) {
- ColumnSchema next = iterator.next();
- if ((null == next.getAggFunction() || next.getAggFunction().isEmpty())) {
- return next;
- }
- }
- }
- return null;
- }
-
- /**
- * Below method will be used to get the columns on which aggregate function is not applied
- *
- * @param columnName parent column name
- * @return child column schema
- */
- public ColumnSchema getTimeseriesChildColBasedByParent(String columnName,
- String timeseriesFunction) {
- Set<ColumnSchema> columnSchemas = parentToNonAggChildMapping.get(columnName);
- if (null != columnSchemas) {
- Iterator<ColumnSchema> iterator = columnSchemas.iterator();
- while (iterator.hasNext()) {
- ColumnSchema next = iterator.next();
- if (timeseriesFunction.equals(next.getTimeSeriesFunction())) {
- return next;
- }
- }
- }
- return null;
- }
-
- /**
- * Below method will be used to get the column schema based on parent column name
- * @param columName
- * parent column name
- * @return child column schema
- */
- public ColumnSchema getChildColByParentColName(String columName) {
- List<ColumnSchema> listOfColumns = childSchema.getListOfColumns();
- for (ColumnSchema columnSchema : listOfColumns) {
- List<ParentColumnTableRelation> parentColumnTableRelations =
- columnSchema.getParentColumnTableRelations();
- if (null != parentColumnTableRelations && parentColumnTableRelations.size() == 1
- && parentColumnTableRelations.get(0).getColumnName().equalsIgnoreCase(columName) &&
- columnSchema.getColumnName().endsWith(columName)) {
- return columnSchema;
- }
- }
- return null;
- }
-
- /**
- * Below method will be used to get the child column schema based on parent name and aggregate
- * function applied on column
- * @param columnName
- * parent column name
- * @param aggFunction
- * aggregate function applied
- * @return child column schema
- */
- public ColumnSchema getAggChildColByParent(String columnName,
- String aggFunction) {
- Set<ColumnSchema> columnSchemas = parentToAggChildMapping.get(columnName);
- if (null != columnSchemas) {
- Iterator<ColumnSchema> iterator = columnSchemas.iterator();
- while (iterator.hasNext()) {
- ColumnSchema next = iterator.next();
- if (null != next.getAggFunction() && next.getAggFunction().equalsIgnoreCase(aggFunction)) {
- return next;
- }
- }
- }
- return null;
- }
-
- /**
- * Below method will be used to get the column schema based on parent column name
- * @param columName
- * parent column name
- * @param timeseriesFunction
- * timeseries function applied on column
- * @return child column schema
- */
- public ColumnSchema getTimeseriesChildColByParent(String columName, String timeseriesFunction) {
- List<ColumnSchema> listOfColumns = childSchema.getListOfColumns();
- for (ColumnSchema columnSchema : listOfColumns) {
- List<ParentColumnTableRelation> parentColumnTableRelations =
- columnSchema.getParentColumnTableRelations();
- if (null != parentColumnTableRelations && parentColumnTableRelations.size() == 1
- && parentColumnTableRelations.get(0).getColumnName().equalsIgnoreCase(columName)
- && timeseriesFunction.equalsIgnoreCase(columnSchema.getTimeSeriesFunction())) {
- return columnSchema;
- }
- }
- return null;
- }
-
- /**
- * Method to prepare mapping of parent to list of aggregation function applied on that column
- * @param listOfColumns
- * child column schema list
- */
- private void fillParentNameToAggregationMapping(List<ColumnSchema> listOfColumns) {
- parentColumnToAggregationsMapping = new HashMap<>();
- for (ColumnSchema column : listOfColumns) {
- if (null != column.getAggFunction() && !column.getAggFunction().isEmpty()) {
- List<ParentColumnTableRelation> parentColumnTableRelations =
- column.getParentColumnTableRelations();
- if (null != parentColumnTableRelations && parentColumnTableRelations.size() == 1) {
- String columnName = column.getParentColumnTableRelations().get(0).getColumnName();
- Set<String> aggFunctions = parentColumnToAggregationsMapping.get(columnName);
- if (null == aggFunctions) {
- aggFunctions = new HashSet<>();
- parentColumnToAggregationsMapping.put(columnName, aggFunctions);
- }
- aggFunctions.add(column.getAggFunction());
- }
- }
- }
- }
-
- /**
- * Below method will be used prepare mapping between parent column to non aggregation function
- * columns
- * @param listOfColumns
- * list of child columns
- */
- private void fillNonAggFunctionColumns(List<ColumnSchema> listOfColumns) {
- parentToNonAggChildMapping = new HashMap<>();
- for (ColumnSchema column : listOfColumns) {
- if (!isTimeseriesDataMap) {
- isTimeseriesDataMap =
- null != column.getTimeSeriesFunction() && !column.getTimeSeriesFunction().isEmpty();
- if (isTimeseriesDataMap) {
- this.ordinal =
- TimeSeriesFunctionEnum.valueOf(column.getTimeSeriesFunction().toUpperCase())
- .getOrdinal();
- }
- }
- if (null == column.getAggFunction() || column.getAggFunction().isEmpty()) {
- fillMappingDetails(column, parentToNonAggChildMapping);
- }
- }
- }
-
- private void fillMappingDetails(ColumnSchema column,
- Map<String, Set<ColumnSchema>> map) {
- List<ParentColumnTableRelation> parentColumnTableRelations =
- column.getParentColumnTableRelations();
- if (null != parentColumnTableRelations && parentColumnTableRelations.size() == 1) {
- String columnName = column.getParentColumnTableRelations().get(0).getColumnName();
- Set<ColumnSchema> columnSchemas = map.get(columnName);
- if (null == columnSchemas) {
- columnSchemas = new HashSet<>();
- map.put(columnName, columnSchemas);
- }
- columnSchemas.add(column);
- }
- }
-
- /**
- * Below method will be used to fill parent to list of aggregation column mapping
- * @param listOfColumns
- * list of child columns
- */
- private void fillAggFunctionColumns(List<ColumnSchema> listOfColumns) {
- parentToAggChildMapping = new HashMap<>();
- for (ColumnSchema column : listOfColumns) {
- if (null != column.getAggFunction() && !column.getAggFunction().isEmpty()) {
- fillMappingDetails(column, parentToAggChildMapping);
- }
- }
- }
-
- public boolean isTimeseriesDataMap() {
- return isTimeseriesDataMap;
- }
-
- /**
- * Below method is to support rollup during loading the data in pre aggregate table
- * In case of timeseries year level table data loading can be done using month level table or any
- * time series level below year level for example day,hour minute, second.
- * @TODO need to handle for pre aggregate table without timeseries
- *
- * @param aggregationDataMapSchema
- * @return whether aggregation data map can be selected or not
- */
- public boolean canSelectForRollup(AggregationDataMapSchema aggregationDataMapSchema) {
- List<ColumnSchema> listOfColumns = childSchema.getListOfColumns();
- for (ColumnSchema column : listOfColumns) {
- List<ParentColumnTableRelation> parentColumnTableRelations =
- column.getParentColumnTableRelations();
- //@TODO handle scenario when aggregate datamap columns is derive from multiple column
- // which is not supported currently
- if (null != parentColumnTableRelations && parentColumnTableRelations.size() == 1) {
- if (null != column.getAggFunction() && !column.getAggFunction().isEmpty()) {
- if (null == aggregationDataMapSchema
- .getAggChildColByParent(parentColumnTableRelations.get(0).getColumnName(),
- column.getAggFunction())) {
- return false;
- }
- } else {
- if (null == aggregationDataMapSchema.getNonAggChildColBasedByParent(
- parentColumnTableRelations.get(0).getColumnName())) {
- return false;
- }
- }
- } else {
- // in case of any expression one column can be derived from multiple column
- // in that case we cannot do rollup so hit the maintable
- return false;
- }
- }
- return true;
- }
-
- public int getOrdinal() {
- return ordinal;
- }
-
- /**
- * Below method will be used to get the aggregation column based on index
- * It will return the first aggregation column found based on index
- * @param searchStartIndex
- * start index
- * @param sortedColumnSchema
- * list of sorted table columns
- * @return found column list
- *
- */
- public ColumnSchema getAggColumnBasedOnIndex(int searchStartIndex,
- List<ColumnSchema> sortedColumnSchema) {
- ColumnSchema columnSchema = null;
- for (int i = searchStartIndex; i < sortedColumnSchema.size(); i++) {
- if (!sortedColumnSchema.get(i).getAggFunction().isEmpty()) {
- columnSchema = sortedColumnSchema.get(i);
- break;
- }
- }
- return columnSchema;
- }
-
- public synchronized Set getAggExpToColumnMapping() {
- return aggExpToColumnMapping;
- }
-
- public synchronized void setAggExpToColumnMapping(Set aggExpToColumnMapping) {
- if (null == this.aggExpToColumnMapping) {
- this.aggExpToColumnMapping = aggExpToColumnMapping;
- }
- }
-
- public DataMapClassProvider getProvider() {
- return isTimeseriesDataMap ?
- DataMapClassProvider.TIMESERIES : DataMapClassProvider.PREAGGREGATE;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- if (!super.equals(o)) return false;
- AggregationDataMapSchema that = (AggregationDataMapSchema) o;
- return that == this;
- }
-
- @Override
- public int hashCode() {
- return super.hashCode();
- }
-}
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 9b9b756..86ce9ba 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -861,21 +861,6 @@ public class CarbonTable implements Serializable, Writable {
return streaming != null && streaming.equalsIgnoreCase("source");
}
- /**
- * whether this table has aggregation DataMap or not
- */
- public boolean hasAggregationDataMap() {
- List<DataMapSchema> dataMapSchemaList = tableInfo.getDataMapSchemaList();
- if (dataMapSchemaList != null && !dataMapSchemaList.isEmpty()) {
- for (DataMapSchema dataMapSchema : dataMapSchemaList) {
- if (dataMapSchema instanceof AggregationDataMapSchema) {
- return true;
- }
- }
- }
- return false;
- }
-
public int getDimensionOrdinalMax() {
return dimensionOrdinalMax;
}
@@ -894,12 +879,7 @@ public class CarbonTable implements Serializable, Writable {
return null;
}
- public boolean isChildDataMap() {
- return null != tableInfo.getParentRelationIdentifiers() && !tableInfo
- .getParentRelationIdentifiers().isEmpty();
- }
-
- public boolean isChildTable() {
+ public boolean isChildTableForMV() {
return null != tableInfo.getFactTable().getTableProperties()
.get(CarbonCommonConstants.PARENT_TABLES) && !tableInfo.getFactTable().getTableProperties()
.get(CarbonCommonConstants.PARENT_TABLES).isEmpty();
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
index 69ff837..ac5ce19 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
@@ -171,9 +171,7 @@ public class DataMapSchema implements Serializable, Writable {
* @return
*/
public boolean isIndexDataMap() {
- if (providerName.equalsIgnoreCase(DataMapClassProvider.PREAGGREGATE.getShortName()) ||
- providerName.equalsIgnoreCase(DataMapClassProvider.TIMESERIES.getShortName()) ||
- providerName.equalsIgnoreCase(DataMapClassProvider.MV.getShortName()) ||
+ if (providerName.equalsIgnoreCase(DataMapClassProvider.MV.getShortName()) ||
ctasQuery != null) {
return false;
} else {
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java
index bd79e08..d7abc4b 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java
@@ -17,8 +17,6 @@
package org.apache.carbondata.core.metadata.schema.table;
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider;
-
public class DataMapSchemaFactory {
public static final DataMapSchemaFactory INSTANCE = new DataMapSchemaFactory();
@@ -29,14 +27,6 @@ public class DataMapSchemaFactory {
* @return data map schema
*/
public DataMapSchema getDataMapSchema(String dataMapName, String providerName) {
- if (providerName.equalsIgnoreCase(DataMapClassProvider.PREAGGREGATE.toString()) || providerName
- .equalsIgnoreCase(DataMapClassProvider.PREAGGREGATE.getClassName())) {
- return new AggregationDataMapSchema(dataMapName, providerName);
- } else if (providerName.equalsIgnoreCase(DataMapClassProvider.TIMESERIES.toString())
- || providerName.equalsIgnoreCase(DataMapClassProvider.TIMESERIES.getClassName())) {
- return new AggregationDataMapSchema(dataMapName, providerName);
- } else {
- return new DataMapSchema(dataMapName, providerName);
- }
+ return new DataMapSchema(dataMapName, providerName);
}
}
diff --git a/core/src/main/java/org/apache/carbondata/core/preagg/AggregateQueryPlan.java b/core/src/main/java/org/apache/carbondata/core/preagg/AggregateQueryPlan.java
deleted file mode 100644
index 4ff89f8..0000000
--- a/core/src/main/java/org/apache/carbondata/core/preagg/AggregateQueryPlan.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.preagg;
-
-import java.util.List;
-
-/**
- * class to maintain the query plan to select the data map tables
- */
-public class AggregateQueryPlan {
-
- /**
- * List of projection columns
- */
- private List<QueryColumn> projectionColumn;
-
- /**
- * list of filter columns
- */
- private List<QueryColumn> filterColumns;
-
- public AggregateQueryPlan(List<QueryColumn> projectionColumn, List<QueryColumn> filterColumns) {
- this.projectionColumn = projectionColumn;
- this.filterColumns = filterColumns;
- }
-
- public List<QueryColumn> getProjectionColumn() {
- return projectionColumn;
- }
-
- public List<QueryColumn> getFilterColumns() {
- return filterColumns;
- }
-}
diff --git a/core/src/main/java/org/apache/carbondata/core/preagg/AggregateTableSelector.java b/core/src/main/java/org/apache/carbondata/core/preagg/AggregateTableSelector.java
deleted file mode 100644
index 5cdbad0..0000000
--- a/core/src/main/java/org/apache/carbondata/core/preagg/AggregateTableSelector.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.preagg;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-
-/**
- * Below class will be used to select the aggregate table based
- * query plan. Rules for selecting the aggregate table is below:
- * 1. Select all aggregate table based on projection
- * 2. select aggregate table based on filter exp,
- * 2. select if aggregate tables based on aggregate columns
- */
-public class AggregateTableSelector {
-
- /**
- * current query plan
- */
- private AggregateQueryPlan aggregateQueryPlan;
-
- /**
- * parent table
- */
- private CarbonTable parentTable;
-
- public AggregateTableSelector(AggregateQueryPlan aggregateQueryPlan, CarbonTable parentTable) {
- this.aggregateQueryPlan = aggregateQueryPlan;
- this.parentTable = parentTable;
- }
-
- /**
- * Below method will be used to select pre aggregate tables based on query plan
- * Rules for selecting the aggregate table is below:
- * 1. Select all aggregate table based on projection
- * 2. select aggregate table based on filter exp,
- * 2. select if aggregate tables based on aggregate columns
- *
- * @return selected pre aggregate table schema
- */
- public List<DataMapSchema> selectPreAggDataMapSchema() {
- List<QueryColumn> projectionColumn = aggregateQueryPlan.getProjectionColumn();
- List<QueryColumn> filterColumns = aggregateQueryPlan.getFilterColumns();
- List<DataMapSchema> dataMapSchemaList = parentTable.getTableInfo().getDataMapSchemaList();
- List<DataMapSchema> selectedDataMapSchema = new ArrayList<>();
- boolean isMatch;
- // match projection columns
- if (null != projectionColumn && !projectionColumn.isEmpty()) {
- for (DataMapSchema dmSchema : dataMapSchemaList) {
- AggregationDataMapSchema aggregationDataMapSchema = (AggregationDataMapSchema) dmSchema;
- isMatch = true;
- for (QueryColumn queryColumn : projectionColumn) {
- ColumnSchema columnSchemaByParentName =
- getColumnSchema(queryColumn, aggregationDataMapSchema);
- if (null == columnSchemaByParentName) {
- isMatch = false;
- break;
- }
- }
- if (isMatch) {
- selectedDataMapSchema.add(dmSchema);
- }
- }
- // if projection column is present but selected table list size is zero then
- if (selectedDataMapSchema.size() == 0) {
- return selectedDataMapSchema;
- }
- }
-
- // match filter columns
- if (null != filterColumns && !filterColumns.isEmpty()) {
- List<DataMapSchema> dmSchemaToIterate =
- selectedDataMapSchema.isEmpty() ? dataMapSchemaList : selectedDataMapSchema;
- selectedDataMapSchema = new ArrayList<>();
- for (DataMapSchema dmSchema : dmSchemaToIterate) {
- isMatch = true;
- for (QueryColumn queryColumn : filterColumns) {
- AggregationDataMapSchema aggregationDataMapSchema = (AggregationDataMapSchema) dmSchema;
- ColumnSchema columnSchemaByParentName =
- getColumnSchema(queryColumn, aggregationDataMapSchema);
- if (null == columnSchemaByParentName) {
- isMatch = false;
- break;
- }
- }
- if (isMatch) {
- selectedDataMapSchema.add(dmSchema);
- }
- }
- // if filter column is present and selection size is zero then return
- if (selectedDataMapSchema.size() == 0) {
- return selectedDataMapSchema;
- }
- }
- return selectedDataMapSchema;
- }
-
- /**
- * Below method will be used to get column schema for projection and
- * filter query column
- *
- * @param queryColumn query column
- * @param aggregationDataMapSchema selected data map schema
- * @return column schema
- */
- private ColumnSchema getColumnSchema(QueryColumn queryColumn,
- AggregationDataMapSchema aggregationDataMapSchema) {
- ColumnSchema columnSchemaByParentName = null;
- if (!queryColumn.getTimeseriesFunction().isEmpty()) {
- columnSchemaByParentName = aggregationDataMapSchema
- .getTimeseriesChildColBasedByParent(queryColumn.getColumnSchema().getColumnName(),
- queryColumn.getTimeseriesFunction());
- } else {
- columnSchemaByParentName = aggregationDataMapSchema
- .getNonAggNonTimeseriesChildColBasedByParent(
- queryColumn.getColumnSchema().getColumnName());
- }
- return columnSchemaByParentName;
- }
-}
diff --git a/core/src/main/java/org/apache/carbondata/core/preagg/QueryColumn.java b/core/src/main/java/org/apache/carbondata/core/preagg/QueryColumn.java
deleted file mode 100644
index 64b4967..0000000
--- a/core/src/main/java/org/apache/carbondata/core/preagg/QueryColumn.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.preagg;
-
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-
-/**
- * column present in query
- */
-public class QueryColumn {
-
- /**
- * parent column schema
- */
- private ColumnSchema columnSchema;
-
- /**
- * is filter column
- */
- private boolean isFilterColumn;
-
- /**
- * timeseries udf applied on column
- */
- private String timeseriesFunction;
-
- public QueryColumn(ColumnSchema columnSchema, boolean isFilterColumn, String timeseriesFunction) {
- this.columnSchema = columnSchema;
- this.isFilterColumn = isFilterColumn;
- this.timeseriesFunction = timeseriesFunction;
- }
-
- public ColumnSchema getColumnSchema() {
- return columnSchema;
- }
-
- public boolean isFilterColumn() {
- return isFilterColumn;
- }
-
- public String getTimeseriesFunction() {
- return timeseriesFunction;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- QueryColumn that = (QueryColumn) o;
- if (isFilterColumn != that.isFilterColumn) {
- return false;
- }
- if (!columnSchema.equals(that.columnSchema)) {
- return false;
- }
- return timeseriesFunction != null ?
- timeseriesFunction.equals(that.timeseriesFunction) :
- that.timeseriesFunction == null;
- }
-
- @Override
- public int hashCode() {
- int result = columnSchema.hashCode();
- result = 31 * result + (timeseriesFunction != null ? timeseriesFunction.hashCode() : 0);
- result = 31 * result + (isFilterColumn ? 1 : 0);
- return result;
- }
-}
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index df5a49e..d3fa33b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -88,9 +88,7 @@ import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry;
-import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.metadata.schema.table.TableSchema;
@@ -2711,38 +2709,6 @@ public final class CarbonUtil {
}
/**
- * Utility function to check whether table has timseries datamap or not
- * @param carbonTable
- * @return timeseries data map present
- */
- public static boolean hasTimeSeriesDataMap(CarbonTable carbonTable) {
- List<DataMapSchema> dataMapSchemaList = carbonTable.getTableInfo().getDataMapSchemaList();
- for (DataMapSchema dataMapSchema : dataMapSchemaList) {
- if (dataMapSchema instanceof AggregationDataMapSchema) {
- if (((AggregationDataMapSchema) dataMapSchema).isTimeseriesDataMap()) {
- return true;
- }
- }
- }
- return false;
- }
-
- /**
- * Utility function to check whether table has aggregation datamap or not
- * @param carbonTable
- * @return timeseries data map present
- */
- public static boolean hasAggregationDataMap(CarbonTable carbonTable) {
- List<DataMapSchema> dataMapSchemaList = carbonTable.getTableInfo().getDataMapSchemaList();
- for (DataMapSchema dataMapSchema : dataMapSchemaList) {
- if (dataMapSchema instanceof AggregationDataMapSchema) {
- return true;
- }
- }
- return false;
- }
-
- /**
* Convert the bytes to base64 encode string
* @param bytes
* @return
@@ -3199,7 +3165,7 @@ public final class CarbonUtil {
SegmentStatusManager segmentStatusManager =
new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier());
SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo =
- segmentStatusManager.getValidAndInvalidSegments(carbonTable.isChildTable());
+ segmentStatusManager.getValidAndInvalidSegments(carbonTable.isChildTableForMV());
List<Segment> validSegments = validAndInvalidSegmentsInfo.getValidSegments();
if (validSegments.isEmpty()) {
return carbonProperties.getFormatVersion();
@@ -3350,22 +3316,6 @@ public final class CarbonUtil {
return UUID.randomUUID().toString();
}
- /**
- * Below method will be used to get the datamap schema name from datamap table name
- * it will split name based on character '_' and get the last name
- * This is only for pre aggregate and timeseries tables
- *
- * @param tableName
- * @return datamapschema name
- */
- public static String getDatamapNameFromTableName(String tableName) {
- int i = tableName.lastIndexOf('_');
- if (i != -1) {
- return tableName.substring(i + 1, tableName.length());
- }
- return null;
- }
-
public static String getIndexServerTempPath(String tablePath, String queryId) {
String tempFolderPath = CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.CARBON_INDEX_SERVER_TEMP_PATH);
diff --git a/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java b/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
index 151eef0..1769e16 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
@@ -26,7 +26,6 @@ import org.apache.carbondata.common.constants.LoggerAction;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.constants.CarbonCommonConstantsInternal;
import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
import org.apache.carbondata.core.exception.InvalidConfigurationException;
@@ -219,12 +218,8 @@ public class SessionParams implements Serializable, Cloneable {
if (!isValid) {
throw new InvalidConfigurationException("Invalid CARBON_INPUT_SEGMENT_IDs");
}
- } else if (key.startsWith(CarbonCommonConstants.VALIDATE_CARBON_INPUT_SEGMENTS)) {
- isValid = true;
} else if (key.equalsIgnoreCase(CarbonCommonConstants.SUPPORT_DIRECT_QUERY_ON_DATAMAP)) {
isValid = true;
- } else if (key.startsWith(CarbonCommonConstantsInternal.QUERY_ON_PRE_AGG_STREAMING)) {
- isValid = true;
} else if (key.startsWith(CarbonCommonConstants.CARBON_DATAMAP_VISIBLE)) {
isValid = true;
} else if (key.startsWith(CarbonCommonConstants.CARBON_LOAD_DATAMAPS_PARALLEL)) {
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
index dfec812..882ee45 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
@@ -41,11 +41,9 @@ import org.apache.carbondata.core.devapi.DictionaryGenerationException;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
@@ -115,7 +113,6 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
for (CarbonColumn col : indexedColumn) {
this.name2Col.put(col.getColName(), col);
}
- String parentTablePath = getAncestorTablePath(carbonTable);
try {
this.name2Converters = new HashMap<>(indexedColumn.size());
@@ -137,7 +134,7 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
dataField.setTimestampFormat(tsFormat);
FieldConverter fieldConverter = FieldEncoderFactory.getInstance()
.createFieldEncoder(dataField, absoluteTableIdentifier, i, nullFormat, null, false,
- localCaches[i], false, parentTablePath, false);
+ localCaches[i], false, carbonTable.getTablePath(), false);
this.name2Converters.put(indexedColumn.get(i).getColName(), fieldConverter);
}
} catch (IOException e) {
@@ -148,22 +145,6 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
this.badRecordLogHolder.setLogged(false);
}
- /**
- * recursively find the ancestor's table path. This is used for dictionary scenario
- * where preagg will use the dictionary of the parent table.
- */
- private String getAncestorTablePath(CarbonTable currentTable) {
- if (!currentTable.isChildDataMap()) {
- return currentTable.getTablePath();
- }
-
- RelationIdentifier parentIdentifier =
- currentTable.getTableInfo().getParentRelationIdentifiers().get(0);
- CarbonTable parentTable = CarbonMetadata.getInstance().getCarbonTable(
- parentIdentifier.getDatabaseName(), parentIdentifier.getTableName());
- return getAncestorTablePath(parentTable);
- }
-
@Override
public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
List<PartitionSpec> partitions) throws IOException {
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
index e3902b3..4cb5385 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
@@ -374,7 +374,7 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa
new SegmentStatusManager(getCarbonTable().getAbsoluteTableIdentifier());
try {
List<Segment> validSegments =
- ssm.getValidAndInvalidSegments(getCarbonTable().isChildTable()).getValidSegments();
+ ssm.getValidAndInvalidSegments(getCarbonTable().isChildTableForMV()).getValidSegments();
for (Segment segment : validSegments) {
deleteDatamapData(segment);
}
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
index 52dfa09..247e856 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
@@ -181,7 +181,7 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> extends DataMapFactor
SegmentStatusManager ssm = new SegmentStatusManager(tableIdentifier);
try {
List<Segment> validSegments =
- ssm.getValidAndInvalidSegments(getCarbonTable().isChildTable()).getValidSegments();
+ ssm.getValidAndInvalidSegments(getCarbonTable().isChildTableForMV()).getValidSegments();
for (Segment segment : validSegments) {
deleteDatamapData(segment);
}
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala
index 0ba6bfc..da38bae 100644
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala
@@ -18,7 +18,7 @@ package org.apache.carbondata.mv.datamap
import scala.collection.JavaConverters._
-import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.{CarbonEnv, SparkSession}
import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias, UnresolvedAttribute}
import org.apache.spark.sql.catalyst.catalog.CatalogTable
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, ScalaUDF}
@@ -53,10 +53,10 @@ class MVAnalyzerRule(sparkSession: SparkSession) extends Rule[LogicalPlan] {
override def apply(plan: LogicalPlan): LogicalPlan = {
var needAnalysis = true
plan.transformAllExpressions {
- // first check if any preAgg scala function is applied it is present is in plan
- // then call is from create preaggregate table class so no need to transform the query plan
+ // first check if any mv UDF is applied it is present is in plan
+ // then call is from create MV so no need to transform the query plan
// TODO Add different UDF name
- case al@Alias(udf: ScalaUDF, name) if name.equalsIgnoreCase("preAgg") =>
+ case al@Alias(udf: ScalaUDF, name) if name.equalsIgnoreCase(CarbonEnv.MV_SKIP_RULE_UDF) =>
needAnalysis = false
al
// in case of query if any unresolve alias is present then wait for plan to be resolved
@@ -75,7 +75,7 @@ class MVAnalyzerRule(sparkSession: SparkSession) extends Rule[LogicalPlan] {
if (p.isInstanceOf[UnresolvedAlias]) {
false
} else {
- p.name.equals("preAggLoad") || p.name.equals("preAgg")
+ p.name.equals(CarbonEnv.MV_SKIP_RULE_UDF)
}
}
if (isPreAggLoad) {
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVDataMapProvider.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVDataMapProvider.scala
index 6499a8f..2c2ad1e 100644
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVDataMapProvider.scala
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVDataMapProvider.scala
@@ -20,7 +20,7 @@ import java.io.IOException
import scala.collection.JavaConverters._
-import org.apache.spark.sql.{CarbonSession, SparkSession}
+import org.apache.spark.sql.{CarbonEnv, CarbonSession, SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.plans.logical.SubqueryAlias
import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand
@@ -121,18 +121,10 @@ class MVDataMapProvider(
val ctasQuery = dataMapSchema.getCtasQuery
if (ctasQuery != null) {
val identifier = dataMapSchema.getRelationIdentifier
- val logicalPlan =
- new FindDataSourceTable(sparkSession).apply(
- SparkSQLUtil.sessionState(sparkSession).catalog.lookupRelation(
- TableIdentifier(identifier.getTableName,
- Some(identifier.getDatabaseName)))) match {
- case s: SubqueryAlias => s.child
- case other => other
- }
- val updatedQuery = new CarbonSpark2SqlParser().addPreAggFunction(ctasQuery)
+ val updatedQuery = new CarbonSpark2SqlParser().addMVSkipFunction(ctasQuery)
val queryPlan = SparkSQLUtil.execute(
sparkSession.sql(updatedQuery).queryExecution.analyzed,
- sparkSession).drop("preAgg")
+ sparkSession).drop("mv")
var isOverwriteTable = false
val isFullRefresh =
if (null != dataMapSchema.getProperties.get("full_refresh")) {
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala
index c28a7a5..b32367b 100644
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala
@@ -68,7 +68,7 @@ object MVHelper {
}
val mvUtil = new MVUtil
mvUtil.validateDMProperty(dmProperties)
- val updatedQuery = new CarbonSpark2SqlParser().addPreAggFunction(queryString)
+ val updatedQuery = new CarbonSpark2SqlParser().addMVSkipFunction(queryString)
val query = sparkSession.sql(updatedQuery)
val logicalPlan = MVHelper.dropDummFuc(query.queryExecution.analyzed)
// if there is limit in MV ctas query string, throw exception, as its not a valid usecase
@@ -136,7 +136,7 @@ object MVHelper {
if (!mainCarbonTable.get.getTableInfo.isTransactionalTable) {
throw new MalformedCarbonCommandException("Unsupported operation on NonTransactional table")
}
- if (mainCarbonTable.get.isChildTable || mainCarbonTable.get.isChildDataMap) {
+ if (mainCarbonTable.get.isChildTableForMV) {
throw new MalformedCarbonCommandException(
"Cannot create Datamap on child table " + mainCarbonTable.get.getTableUniqueName)
}
@@ -242,7 +242,6 @@ object MVHelper {
tableProperties,
None,
isAlterFlow = false,
- isPreAggFlow = false,
None)
val tablePath = if (dmProperties.contains("path")) {
@@ -403,8 +402,10 @@ object MVHelper {
private def dropDummyExp(exps: Seq[NamedExpression]) = {
exps.map {
- case al@Alias(udf: ScalaUDF, name) if name.equalsIgnoreCase("preAgg") => None
- case attr: AttributeReference if attr.name.equalsIgnoreCase("preAgg") => None
+ case al@Alias(udf: ScalaUDF, name) if name.equalsIgnoreCase(CarbonEnv.MV_SKIP_RULE_UDF) =>
+ None
+ case attr: AttributeReference if attr.name.equalsIgnoreCase(CarbonEnv.MV_SKIP_RULE_UDF) =>
+ None
case other => Some(other)
}.filter(_.isDefined).map(_.get)
}
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/SummaryDatasetCatalog.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/SummaryDatasetCatalog.scala
index ea0bfea..645b304 100644
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/SummaryDatasetCatalog.scala
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/SummaryDatasetCatalog.scala
@@ -107,8 +107,7 @@ private[mv] class SummaryDatasetCatalog(sparkSession: SparkSession)
*/
private[mv] def registerSchema(dataMapSchema: DataMapSchema): Unit = {
writeLock {
- // TODO Add mvfunction here, don't use preagg function
- val updatedQuery = parser.addPreAggFunction(dataMapSchema.getCtasQuery)
+ val updatedQuery = parser.addMVSkipFunction(dataMapSchema.getCtasQuery)
val query = sparkSession.sql(updatedQuery)
val planToRegister = MVHelper.dropDummFuc(query.queryExecution.analyzed)
val modularPlan =
diff --git a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVCreateTestCase.scala b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVCreateTestCase.scala
index 16ca086..8972068 100644
--- a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVCreateTestCase.scala
+++ b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVCreateTestCase.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.{CarbonEnv, Row}
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
@@ -813,7 +814,6 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
test("jira carbondata-2560") {
- sql("drop datamap if exists MV_exp1")
sql("drop datamap if exists MV_exp2")
sql("create datamap MV_exp1 using 'mv' as select empname, sum(utilization) from fact_table1 group by empname")
intercept[UnsupportedOperationException] {
@@ -884,7 +884,6 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
test("test create datamap with streaming table") {
sql("drop datamap if exists dm_stream_test1")
sql("drop datamap if exists dm_stream_bloom")
- sql("drop datamap if exists dm_stream_PreAggMax")
sql("drop table if exists fact_streaming_table1")
sql(
"""
@@ -902,9 +901,6 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
| DMProperties('INDEX_COLUMNS'='empname,deptname', 'BLOOM_SIZE'='640000')
""".stripMargin)
- sql("create datamap dm_stream_PreAggMax on table fact_streaming_table1 using 'preaggregate' " +
- "as select empname,max(salary) as max from fact_streaming_table1 group by empname")
-
val exception_tb_mv: Exception = intercept[Exception] {
sql("create datamap dm_stream_test1 using 'mv' as select empname, sum(utilization) from " +
"fact_streaming_table1 group by empname")
diff --git a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/TestAllOperationsOnMV.scala b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/TestAllOperationsOnMV.scala
index f140e5d..d0f9ce2 100644
--- a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/TestAllOperationsOnMV.scala
+++ b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/TestAllOperationsOnMV.scala
@@ -203,11 +203,10 @@ class TestAllOperationsOnMV extends QueryTest with BeforeAndAfterEach {
sql("rebuild datamap dm1")
intercept[UnsupportedOperationException] {
sql("insert into dm1_table select 2")
- }.getMessage.contains("Cannot insert/load data directly into pre-aggregate/child table")
+ }.getMessage.contains("Cannot insert data directly into MV table")
sql("drop table IF EXISTS maintable")
}
-
test("test drop datamap with tablename") {
sql("drop table IF EXISTS maintable")
sql("create table maintable(name string, c_code int, price int) stored by 'carbondata'")
@@ -317,9 +316,6 @@ class TestAllOperationsOnMV extends QueryTest with BeforeAndAfterEach {
sql("drop datamap if exists dm1 ")
sql("create datamap dm1 using 'mv' as select name, price from maintable")
intercept[Exception] {
- sql("create datamap dm_agg on table dm1_table using 'preaggregate' as select maintable_name, sum(maintable_price) from dm1_table group by maintable_name")
- }.getMessage.contains("Cannot create DataMap on child table default.dm1_table")
- intercept[Exception] {
sql("create datamap dm_agg using 'mv' as select maintable_name, sum(maintable_price) from dm1_table group by maintable_name")
}.getMessage.contains("Cannot create DataMap on child table default.dm1_table")
}
@@ -420,17 +416,6 @@ class TestAllOperationsOnMV extends QueryTest with BeforeAndAfterEach {
sql("drop table IF EXISTS maintable")
}
- test("test global dictionary inherited from parent table - Preaggregate") {
- sql("drop table IF EXISTS maintable")
- sql("create table maintable(name string, c_code int, price int) stored by 'carbondata' tblproperties('dictionary_include'='name')")
- sql("insert into table maintable select 'abc',21,2000")
- sql("drop datamap if exists dm ")
- sql("create datamap dm on table maintable using 'preaggregate' as select name, sum(price) from maintable group by name")
- checkExistence(sql("describe formatted maintable_dm"), true, "Global Dictionary maintable_name")
- checkAnswer(sql("select name, sum(price) from maintable group by name"), Seq(Row("abc", 2000)))
- sql("drop table IF EXISTS maintable")
- }
-
test("test preagg and mv") {
sql("drop table IF EXISTS maintable")
sql("create table maintable(name string, c_code int, price int) stored by 'carbondata'")
@@ -438,10 +423,8 @@ class TestAllOperationsOnMV extends QueryTest with BeforeAndAfterEach {
sql("drop datamap if exists dm_mv ")
sql("create datamap dm_mv using 'mv' as select name, sum(price) from maintable group by name")
sql("drop datamap if exists dm_pre ")
- sql("create datamap dm_pre on table maintable using 'preaggregate' as select name, sum(price) from maintable group by name")
sql("insert into table maintable select 'abcd',21,20002")
checkAnswer(sql("select count(*) from dm_mv_table"), Seq(Row(2)))
- checkAnswer(sql("select count(*) from maintable_dm_pre"), Seq(Row(2)))
sql("drop table IF EXISTS maintable")
}
@@ -456,28 +439,13 @@ class TestAllOperationsOnMV extends QueryTest with BeforeAndAfterEach {
sql("drop table IF EXISTS maintable")
}
- test("test inverted index & no-inverted index inherited from parent table - Preaggregate") {
- sql("drop table IF EXISTS maintable")
- sql("create table maintable(name string, c_code int, price int) stored by 'carbondata' tblproperties('sort_columns'='name', 'inverted_index'='name','sort_scope'='local_sort')")
- sql("insert into table maintable select 'abc',21,2000")
- sql("drop datamap if exists dm ")
- sql("create datamap dm on table maintable using 'preaggregate' as select name, sum(price) from maintable group by name")
- checkExistence(sql("describe formatted maintable_dm"), true, "Inverted Index Columns maintable_name")
- checkAnswer(sql("select name, sum(price) from maintable group by name"), Seq(Row("abc", 2000)))
- sql("drop table IF EXISTS maintable")
- }
-
test("test column compressor on preagg and mv") {
sql("drop table IF EXISTS maintable")
sql("create table maintable(name string, c_code int, price int) stored by 'carbondata' tblproperties('carbon.column.compressor'='zstd')")
sql("insert into table maintable select 'abc',21,2000")
- sql("drop datamap if exists dm_pre ")
- sql("create datamap dm_pre on table maintable using 'preaggregate' as select name, sum(price) from maintable group by name")
- var dataMapTable = CarbonMetadata.getInstance().getCarbonTable(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "maintable_dm_pre")
- assert(dataMapTable.getTableInfo.getFactTable.getTableProperties.get(CarbonCommonConstants.COMPRESSOR).equalsIgnoreCase("zstd"))
sql("drop datamap if exists dm_mv ")
sql("create datamap dm_mv on table maintable using 'mv' as select name, sum(price) from maintable group by name")
- dataMapTable = CarbonMetadata.getInstance().getCarbonTable(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "dm_mv_table")
+ val dataMapTable = CarbonMetadata.getInstance().getCarbonTable(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "dm_mv_table")
assert(dataMapTable.getTableInfo.getFactTable.getTableProperties.get(CarbonCommonConstants.COMPRESSOR).equalsIgnoreCase("zstd"))
sql("drop table IF EXISTS maintable")
}
@@ -486,9 +454,6 @@ class TestAllOperationsOnMV extends QueryTest with BeforeAndAfterEach {
sql("drop table IF EXISTS maintable")
sql("create table maintable(name string, c_code int, price int) stored by 'carbondata' tblproperties('sort_columns'='name')")
sql("insert into table maintable select 'abc',21,2000")
- sql("drop datamap if exists dm_pre ")
- sql("create datamap dm_pre on table maintable using 'preaggregate' as select name, sum(price) from maintable group by name")
- checkExistence(sql("describe formatted maintable_dm_pre"), true, "Sort Scope LOCAL_SORT")
sql("create datamap dm_mv on table maintable using 'mv' as select name, sum(price) from maintable group by name")
checkExistence(sql("describe formatted dm_mv_table"), true, "Sort Scope LOCAL_SORT")
sql("drop table IF EXISTS maintable")
@@ -499,9 +464,6 @@ class TestAllOperationsOnMV extends QueryTest with BeforeAndAfterEach {
sql("create table maintable(name string, c_code int, price int) stored by 'carbondata' tblproperties('sort_scope'='no_sort','sort_columns'='name', 'inverted_index'='name')")
sql("insert into table maintable select 'abc',21,2000")
checkExistence(sql("describe formatted maintable"), true, "Inverted Index Columns name")
- sql("drop datamap if exists dm_pre ")
- sql("create datamap dm_pre on table maintable using 'preaggregate' as select name, sum(price) from maintable group by name")
- checkExistence(sql("describe formatted maintable_dm_pre"), true, "Inverted Index Columns maintable_name")
sql("create datamap dm_mv on table maintable using 'mv' as select name, sum(price) from maintable group by name")
checkExistence(sql("describe formatted dm_mv_table"), true, "Inverted Index Columns maintable_name")
sql("drop table IF EXISTS maintable")
diff --git a/docs/datamap-developer-guide.md b/docs/datamap-developer-guide.md
index e1fa355..d1748c1 100644
--- a/docs/datamap-developer-guide.md
+++ b/docs/datamap-developer-guide.md
@@ -26,9 +26,7 @@ Currently, there are two types of DataMap supported:
### DataMap Provider
When user issues `CREATE DATAMAP dm ON TABLE main USING 'provider'`, the corresponding DataMapProvider implementation will be created and initialized.
Currently, the provider string can be:
-1. preaggregate: A type of MVDataMap that do pre-aggregate of single table
-2. timeseries: A type of MVDataMap that do pre-aggregate based on time dimension of the table
-3. class name IndexDataMapFactory implementation: Developer can implement new type of IndexDataMap by extending IndexDataMapFactory
+1. class name IndexDataMapFactory implementation: Developer can implement new type of IndexDataMap by extending IndexDataMapFactory
When user issues `DROP DATAMAP dm ON TABLE main`, the corresponding DataMapProvider interface will be called.
diff --git a/docs/datamap/datamap-management.md b/docs/datamap/datamap-management.md
index f910559..768ad06 100644
--- a/docs/datamap/datamap-management.md
+++ b/docs/datamap/datamap-management.md
@@ -25,7 +25,6 @@
- [DataMap Related Commands](#datamap-related-commands)
- [Explain](#explain)
- [Show DataMap](#show-datamap)
- - [Compaction on DataMap](#compaction-on-datamap)
@@ -47,8 +46,6 @@ Currently, there are 5 DataMap implementations in CarbonData.
| DataMap Provider | Description | DMPROPERTIES | Management |
| ---------------- | ---------------------------------------- | ---------------------------------------- | ---------------- |
-| preaggregate | single table pre-aggregate table | No DMPROPERTY is required | Automatic |
-| timeseries | time dimension rollup table | event_time, xx_granularity, please refer to [Timeseries DataMap](./timeseries-datamap-guide.md) | Automatic |
| mv | multi-table pre-aggregate table | No DMPROPERTY is required | Manual/Automatic |
| lucene | lucene indexing for text column | index_columns to specifying the index columns | Automatic |
| bloomfilter | bloom filter for high cardinality column, geospatial column | index_columns to specifying the index columns | Automatic |
@@ -136,17 +133,9 @@ Scan Table: default.datamap1_table
There is a SHOW DATAMAPS command, when this is issued, system will read all datamap from *system* folder and print all information on screen. The current information includes:
- DataMapName
-- DataMapProviderName like mv, preaggreagte, timeseries, etc
+- DataMapProviderName like mv
- Associated Table
- DataMap Properties
- DataMap status (ENABLED/DISABLED)
- Sync Status - which displays Last segment Id of main table synced with datamap table and its load
end time (Applicable only for mv datamap)
-
-### Compaction on DataMap
-
-This feature applies for preaggregate datamap only
-
-Running Compaction command (`ALTER TABLE COMPACT`) on main table will **not automatically** compact the pre-aggregate tables created on the main table. User need to run Compaction command separately on each pre-aggregate table to compact them.
-
-Compaction is an optional operation for pre-aggregate table. If compaction is performed on main table but not performed on pre-aggregate table, all queries still can benefit from pre-aggregate tables. To further improve the query performance, compaction on pre-aggregate tables can be triggered to merge the segments and files in the pre-aggregate tables.
diff --git a/docs/datamap/preaggregate-datamap-guide.md b/docs/datamap/preaggregate-datamap-guide.md
deleted file mode 100644
index 5369bb7..0000000
--- a/docs/datamap/preaggregate-datamap-guide.md
+++ /dev/null
@@ -1,275 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to you under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# CarbonData Pre-aggregate DataMap
-
-* [Quick Example](#quick-example)
-* [DataMap Management](#datamap-management)
-* [Pre-aggregate Table](#preaggregate-datamap-introduction)
-* [Loading Data](#loading-data)
-* [Querying Data](#querying-data)
-* [Compaction](#compacting-pre-aggregate-tables)
-* [Data Management](#data-management-with-pre-aggregate-tables)
-
-## Quick example
-Download and unzip spark-2.2.0-bin-hadoop2.7.tgz, and export $SPARK_HOME
-
-Package carbon jar, and copy assembly/target/scala-2.11/carbondata_2.11-x.x.x-SNAPSHOT-shade-hadoop2.7.2.jar to $SPARK_HOME/jars
-```shell
-mvn clean package -DskipTests -Pspark-2.2
-```
-
-Start spark-shell in new terminal, type :paste, then copy and run the following code.
-```scala
- import java.io.File
- import org.apache.spark.sql.{CarbonEnv, SparkSession}
- import org.apache.spark.sql.CarbonSession._
- import org.apache.spark.sql.streaming.{ProcessingTime, StreamingQuery}
- import org.apache.carbondata.core.util.path.CarbonStorePath
-
- val warehouse = new File("./warehouse").getCanonicalPath
- val metastore = new File("./metastore").getCanonicalPath
-
- val spark = SparkSession
- .builder()
- .master("local")
- .appName("preAggregateExample")
- .config("spark.sql.warehouse.dir", warehouse)
- .getOrCreateCarbonSession(warehouse, metastore)
-
- spark.sparkContext.setLogLevel("ERROR")
-
- // drop table if exists previously
- spark.sql(s"DROP TABLE IF EXISTS sales")
-
- // Create main table
- spark.sql(
- s"""
- | CREATE TABLE sales (
- | user_id string,
- | country string,
- | quantity int,
- | price bigint)
- | STORED AS carbondata
- """.stripMargin)
-
- // Create pre-aggregate table on the main table
- // If main table already have data, following command
- // will trigger one immediate load to the pre-aggregate table
- spark.sql(
- s"""
- | CREATE DATAMAP agg_sales
- | ON TABLE sales
- | USING "preaggregate"
- | AS
- | SELECT country, sum(quantity), avg(price)
- | FROM sales
- | GROUP BY country
- """.stripMargin)
-
- import spark.implicits._
- import org.apache.spark.sql.SaveMode
- import scala.util.Random
-
- // Load data to the main table, it will also
- // trigger immediate load to pre-aggregate table.
- // These two loading operation is carried out in a
- // transactional manner, meaning that the whole
- // operation will fail if one of the loading fails
- val r = new Random()
- spark.sparkContext.parallelize(1 to 10)
- .map(x => ("ID." + r.nextInt(100000), "country" + x % 8, x % 50, x % 60))
- .toDF("user_id", "country", "quantity", "price")
- .write
- .format("carbondata")
- .option("tableName", "sales")
- .option("compress", "true")
- .mode(SaveMode.Append)
- .save()
-
- spark.sql(
- s"""
- |SELECT country, sum(quantity), avg(price)
- | from sales GROUP BY country
- """.stripMargin).show
-
- spark.stop
-```
-
-#### DataMap Management
-DataMap can be created using following DDL
- ```
- CREATE DATAMAP [IF NOT EXISTS] datamap_name
- ON TABLE main_table
- USING "datamap_provider"
- DMPROPERTIES ('key'='value', ...)
- AS
- SELECT statement
- ```
-The string followed by USING is called DataMap Provider, in this version CarbonData supports two
-kinds of DataMap:
-1. preaggregate, for pre-aggregate table. Pre-Aggregate table supports two values for DMPROPERTIES.
- a. 'path' is used to specify the store location of the datamap.('path'='/location/').
- b. 'partitioning' when set to false enables user to disable partitioning of the datamap.
- Default value is true for this property.
-2. timeseries, for timeseries roll-up table. Please refer to [Timeseries DataMap](./timeseries-datamap-guide.md)
-
-DataMap can be dropped using following DDL
- ```
- DROP DATAMAP [IF EXISTS] datamap_name
- ON TABLE main_table
- ```
-To show all DataMaps created, use:
- ```
- SHOW DATAMAP
- ON TABLE main_table
- ```
-It will show all DataMaps created on main table.
-
-
-## Preaggregate DataMap Introduction
- Pre-aggregate tables are created as DataMaps and managed as tables internally by CarbonData.
- User can create as many pre-aggregate datamaps required to improve query performance,
- provided the storage requirements and loading speeds are acceptable.
-
- Once pre-aggregate datamaps are created, CarbonData's SparkSQL optimizer extension supports to
- select the most efficient pre-aggregate datamap and rewrite the SQL to query against the selected
- datamap instead of the main table. Since the data size of pre-aggregate datamap is smaller,
- user queries are much faster. In our previous experience, we have seen 5X to 100X times faster
- in production SQLs.
-
- For instance, main table called **sales** which is defined as
-
- ```
- CREATE TABLE sales (
- order_time timestamp,
- user_id string,
- sex string,
- country string,
- quantity int,
- price bigint)
- STORED AS carbondata
- ```
-
- User can create pre-aggregate tables using the Create DataMap DDL
-
- ```
- CREATE DATAMAP agg_sales
- ON TABLE sales
- USING "preaggregate"
- AS
- SELECT country, sex, sum(quantity), avg(price)
- FROM sales
- GROUP BY country, sex
- ```
- **NOTE**:
- * Preaggregate datamap is deprecated and it is replaced by MV datamap.
- Please refer [CarbonData MV DataMap](./mv-datamap-guide.md) for more info.
-
-#### Functions supported in pre-aggregate table
-
-| Function | Rollup supported |
-|----------|:----------------:|
-| SUM |Yes |
-| AVG |Yes |
-| MAX |Yes |
-| MIN |Yes |
-| COUNT |Yes |
-
-
-#### How pre-aggregate tables are selected
-When a user query is submitted, during query planning phase, CarbonData will collect all matched
-pre-aggregate tables as candidates according to Relational Algebra transformation rules. Then, the
-best pre-aggregate table for this query will be selected among the candidates based on cost.
-For simplicity, current cost estimation is based on the data size of the pre-aggregate table. (We
-assume that query will be faster on smaller table)
-
-For the main table **sales** and pre-aggregate table **agg_sales** created above, following queries
-```
-SELECT country, sex, sum(quantity), avg(price) from sales GROUP BY country, sex
-
-SELECT sex, sum(quantity) from sales GROUP BY sex
-
-SELECT avg(price), country from sales GROUP BY country
-```
-
-will be transformed by CarbonData's query planner to query against pre-aggregate table
-**agg_sales** instead of the main table **sales**
-
-However, for following queries
-```
-SELECT user_id, country, sex, sum(quantity), avg(price) from sales GROUP BY user_id, country, sex
-
-SELECT sex, avg(quantity) from sales GROUP BY sex
-
-SELECT country, max(price) from sales GROUP BY country
-```
-
-will query against main table **sales** only, because it does not satisfy pre-aggregate table
-selection logic.
-
-## Loading data
-For existing table with loaded data, data load to pre-aggregate table will be triggered by the
-CREATE DATAMAP statement when user creates the pre-aggregate table. For incremental loads after
-aggregates tables are created, loading data to main table triggers the load to pre-aggregate tables
-once main table loading is complete.
-
-These loads are transactional
-meaning that data on main table and pre-aggregate tables are only visible to the user after all
-tables are loaded successfully, if one of these loads fails, new data are not visible in all tables
-as if the load operation is not happened.
-
-## Querying data
-As a technique for query acceleration, Pre-aggregate tables cannot be queried directly.
-Queries are to be made on main table. While doing query planning, internally CarbonData will check
-associated pre-aggregate tables with the main table, and do query plan transformation accordingly.
-
-User can verify whether a query can leverage pre-aggregate table or not by executing `EXPLAIN`
-command, which will show the transformed logical plan, and thus user can check whether pre-aggregate
-table is selected.
-
-
-## Compacting pre-aggregate tables
-Running Compaction command (`ALTER TABLE COMPACT`) on main table will **not automatically**
-compact the pre-aggregate tables created on the main table. User need to run Compaction command
-separately on each pre-aggregate table to compact them.
-
-Compaction is an optional operation for pre-aggregate table. If compaction is performed on
-main table but not performed on pre-aggregate table, all queries still can benefit from
-pre-aggregate tables. To further improve the query performance, compaction on pre-aggregate tables
-can be triggered to merge the segments and files in the pre-aggregate tables.
-
-## Data Management with pre-aggregate tables
-In current implementation, data consistency needs to be maintained for both main table and pre-aggregate
-tables. Once there is pre-aggregate table created on the main table, following command on the main
-table
-is not supported:
-1. Data management command: `UPDATE/DELETE/DELETE SEGMENT`.
-2. Schema management command: `ALTER TABLE DROP COLUMN`, `ALTER TABLE CHANGE DATATYPE`,
-`ALTER TABLE RENAME`. Note that adding a new column is supported, and for dropping columns and
-change datatype command, CarbonData will check whether it will impact the pre-aggregate table, if
- not, the operation is allowed, otherwise operation will be rejected by throwing exception.
-3. Partition management command: `ALTER TABLE ADD/DROP PARTITION`
-4. Complex Datatypes for preaggregate is not supported.
-
-However, there is still way to support these operations on main table, in current CarbonData
-release, user can do as following:
-1. Remove the pre-aggregate table by `DROP DATAMAP` command
-2. Carry out the data management operation on main table
-3. Create the pre-aggregate table again by `CREATE DATAMAP` command
-Basically, user can manually trigger the operation by re-building the datamap.
-
diff --git a/docs/datamap/timeseries-datamap-guide.md b/docs/datamap/timeseries-datamap-guide.md
deleted file mode 100644
index d3357fa..0000000
--- a/docs/datamap/timeseries-datamap-guide.md
+++ /dev/null
@@ -1,154 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to you under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# CarbonData Timeseries DataMap
-
-* [Timeseries DataMap Introduction](#timeseries-datamap-introduction-alpha-feature)
-* [Compaction](#compacting-timeseries-datamp)
-* [Data Management](#data-management-on-timeseries-datamap)
-
-## Timeseries DataMap Introduction (Alpha Feature)
-Timeseries DataMap is a pre-aggregate table implementation based on 'pre-aggregate' DataMap.
-Difference is that Timeseries DataMap has built-in understanding of time hierarchy and
-levels: year, month, day, hour, minute, so that it supports automatic roll-up in time dimension
-for query.
-
-**CAUTION:** Current version of CarbonData does not support roll-up.It will be implemented in future versions.
-
-The data loading, querying, compaction command and its behavior is the same as preaggregate DataMap.
-Please refer to [Pre-aggregate DataMap](./preaggregate-datamap-guide.md)
-for more information.
-
-To use this datamap, user can create multiple timeseries datamap on the main table which has
-a *event_time* column, one datamap for one time granularity.
-
-For example, below statement effectively create multiple pre-aggregate tables on main table called
-**timeseries**
-
-```
-CREATE DATAMAP agg_year
-ON TABLE sales
-USING "timeseries"
-DMPROPERTIES (
- 'event_time'='order_time',
- 'year_granularity'='1',
-) AS
-SELECT order_time, country, sex, sum(quantity), max(quantity), count(user_id), sum(price),
- avg(price) FROM sales GROUP BY order_time, country, sex
-
-CREATE DATAMAP agg_month
-ON TABLE sales
-USING "timeseries"
-DMPROPERTIES (
- 'event_time'='order_time',
- 'month_granularity'='1',
-) AS
-SELECT order_time, country, sex, sum(quantity), max(quantity), count(user_id), sum(price),
- avg(price) FROM sales GROUP BY order_time, country, sex
-
-CREATE DATAMAP agg_day
-ON TABLE sales
-USING "timeseries"
-DMPROPERTIES (
- 'event_time'='order_time',
- 'day_granularity'='1',
-) AS
-SELECT order_time, country, sex, sum(quantity), max(quantity), count(user_id), sum(price),
- avg(price) FROM sales GROUP BY order_time, country, sex
-
-CREATE DATAMAP agg_sales_hour
-ON TABLE sales
-USING "timeseries"
-DMPROPERTIES (
- 'event_time'='order_time',
- 'hour_granularity'='1',
-) AS
-SELECT order_time, country, sex, sum(quantity), max(quantity), count(user_id), sum(price),
- avg(price) FROM sales GROUP BY order_time, country, sex
-
-CREATE DATAMAP agg_minute
-ON TABLE sales
-USING "timeseries"
-DMPROPERTIES (
- 'event_time'='order_time',
- 'minute_granularity'='1',
-) AS
-SELECT order_time, country, sex, sum(quantity), max(quantity), count(user_id), sum(price),
- avg(price) FROM sales GROUP BY order_time, country, sex
-```
-
-For querying timeseries data, Carbondata has builtin support for following time related UDF
-
-```
-timeseries(timeseries column name, 'aggregation level')
-```
-```
-SELECT timeseries(order_time, 'hour'), sum(quantity) FROM sales GROUP BY timeseries(order_time,'hour')
-```
-
-It is **not necessary** to create pre-aggregate tables for each granularity unless required for
-query.
-
-For Example: For main table **sales** , if following timeseries datamaps were created for day
-level and hour level pre-aggregate
-
-```
-CREATE DATAMAP agg_day
-ON TABLE sales
-USING "timeseries"
-DMPROPERTIES (
- 'event_time'='order_time',
- 'day_granularity'='1',
-) AS
-SELECT order_time, country, sex, sum(quantity), max(quantity), count(user_id), sum(price),
- avg(price) FROM sales GROUP BY order_time, country, sex
-
-CREATE DATAMAP agg_sales_hour
-ON TABLE sales
-USING "timeseries"
-DMPROPERTIES (
- 'event_time'='order_time',
- 'hour_granularity'='1',
-) AS
-SELECT order_time, country, sex, sum(quantity), max(quantity), count(user_id), sum(price),
- avg(price) FROM sales GROUP BY order_time, country, sex
-```
-
-Queries like below will not be rolled-up and hit the main table
-```
-Select timeseries(order_time, 'month'), sum(quantity) from sales group by timeseries(order_time,
- 'month')
-
-Select timeseries(order_time, 'year'), sum(quantity) from sales group by timeseries(order_time,
- 'year')
-```
-
-NOTE (**RESTRICTION**):
-* Only value of 1 is supported for hierarchy levels. Other hierarchy levels will be supported in
-the future CarbonData release.
-* timeseries datamap for the desired levels needs to be created one after the other
-* timeseries datamaps created for each level needs to be dropped separately
-
-
-## Compacting timeseries datamp
-Refer to Compaction section in [preaggregation datamap](./preaggregate-datamap-guide.md).
-Same applies to timeseries datamap.
-
-## Data Management on timeseries datamap
-Refer to Data Management section in [preaggregation datamap](./preaggregate-datamap-guide.md).
-Same applies to timeseries datamap.
-
diff --git a/docs/introduction.md b/docs/introduction.md
index 037a296..7c11718 100644
--- a/docs/introduction.md
+++ b/docs/introduction.md
@@ -71,14 +71,6 @@ CarbonData has rich set of features to support various use cases in Big Data ana
### DataMaps
-- ##### Pre-Aggregate
-
- CarbonData has concept of datamaps to assist in pruning of data while querying so that performance is faster.Pre Aggregate tables are kind of datamaps which can improve the query performance by order of magnitude.CarbonData will automatically pre-aggregate the incremental data and re-write the query to automatically fetch from the most appropriate pre-aggregate table to serve the query faster.
-
-- ##### Time Series
-
- CarbonData has built in understanding of time order(Year, month,day,hour, minute,second). Time series is a pre-aggregate table which can automatically roll-up the data to the desired level during incremental load and serve the query from the most appropriate pre-aggregate table.
-
- ##### Bloom filter
CarbonData supports bloom filter as a datamap in order to quickly and efficiently prune the data for scanning and acheive faster query performance.
@@ -89,7 +81,7 @@ CarbonData has rich set of features to support various use cases in Big Data ana
- ##### MV (Materialized Views)
- MVs are kind of pre-aggregate tables which can support efficent query re-write and processing.CarbonData provides MV which can rewrite query to fetch from any table(including non-carbondata tables). Typical usecase is to store the aggregated data of a non-carbondata fact table into carbondata and use mv to rewrite the query to fetch from carbondata.
+ MVs are kind of pre-aggregate and pre-join tables which can support efficient query re-write and processing.CarbonData provides MV which can rewrite query to fetch from any table(including non-carbondata tables). Typical usecase is to store the aggregated data of a non-carbondata fact table into carbondata and use mv to rewrite the query to fetch from carbondata.
### Streaming
diff --git a/docs/language-manual.md b/docs/language-manual.md
index 79aad00..d85c358 100644
--- a/docs/language-manual.md
+++ b/docs/language-manual.md
@@ -27,8 +27,6 @@ CarbonData has its own parser, in addition to Spark's SQL Parser, to parse and p
- [DataMaps](./datamap/datamap-management.md)
- [Bloom](./datamap/bloomfilter-datamap-guide.md)
- [Lucene](./datamap/lucene-datamap-guide.md)
- - [Pre-Aggregate](./datamap/preaggregate-datamap-guide.md)
- - [Time Series](./datamap/timeseries-datamap-guide.md)
- Materialized Views (MV)
- [Streaming](./streaming-guide.md)
- Data Manipulation Statements
diff --git a/docs/presto-guide.md b/docs/presto-guide.md
index c5bcd76..483585f 100644
--- a/docs/presto-guide.md
+++ b/docs/presto-guide.md
@@ -294,5 +294,5 @@ carbondata files.
### Supported features of presto carbon
Presto carbon only supports reading the carbon table which is written by spark carbon or carbon SDK.
During reading, it supports the non-distributed datamaps like block datamap and bloom datamap.
-It doesn't support MV datamap and Pre-aggregate datamap as it needs query plan to be changed and presto does not allow it.
+It doesn't support Materialized View as it needs query plan to be changed and presto does not allow it.
Also Presto carbon supports streaming segment read from streaming table created by spark.
diff --git a/docs/streaming-guide.md b/docs/streaming-guide.md
index b993f82..b66fce4 100644
--- a/docs/streaming-guide.md
+++ b/docs/streaming-guide.md
@@ -273,8 +273,8 @@ ALTER TABLE streaming_table COMPACT 'close_streaming'
## Constraint
1. reject set streaming property from true to false.
2. reject UPDATE/DELETE command on the streaming table.
-3. reject create pre-aggregation DataMap on the streaming table.
-4. reject add the streaming property on the table with pre-aggregation DataMap.
+3. reject create MV on the streaming table.
+4. reject add the streaming property on the table with MV DataMap.
5. if the table has dictionary columns, it will not support concurrent data loading.
6. block delete "streaming" segment while the streaming ingestion is running.
7. block drop the streaming table while the streaming ingestion is running.
diff --git a/docs/usecases.md b/docs/usecases.md
index 8ff4975..c270eef 100644
--- a/docs/usecases.md
+++ b/docs/usecases.md
@@ -189,8 +189,7 @@ Concurrent queries can be more due to the BI dashboard
- Use table block size as 128MB so that pruning is more effective
- Use global sort mode so that the data to be fetched are grouped together
-- Create pre-aggregate tables for non timestamp based group by queries
-- For queries containing group by date, create timeseries based Datamap(pre-aggregate) tables so that the data is rolled up during creation and fetch is faster
+- Create Materialized View for aggregation queries
- Reduce the Spark shuffle partitions.(In our configuration on 14 node cluster, it was reduced to 35 from default of 200)
- Enable global dictionary for columns which have less cardinalities. Aggregation can be done on encoded data, there by improving the performance
- For columns whose cardinality is high,enable the local dictionary so that store size is less and can take dictionary benefit for scan
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/MVDataMapExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/MVDataMapExample.scala
index fbff90b..2600ab9 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/MVDataMapExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/MVDataMapExample.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.SparkSession
import org.apache.carbondata.examples.util.ExampleUtils
/**
- * This example is for pre-aggregate tables.
+ * This example is for Materialized View.
*/
object MVDataMapExample {
@@ -42,7 +42,6 @@ object MVDataMapExample {
+ "../../../..").getCanonicalPath
val testData = s"$rootPath/integration/spark-common-test/src/test/resources/sample.csv"
- // 1. simple usage for Pre-aggregate tables creation and query
spark.sql("DROP TABLE IF EXISTS mainTable")
spark.sql("DROP TABLE IF EXISTS dimtable")
spark.sql(
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/PreAggregateDataMapExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/PreAggregateDataMapExample.scala
deleted file mode 100644
index b008bd4..0000000
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/PreAggregateDataMapExample.scala
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import java.io.File
-
-import org.apache.spark.sql.{SaveMode, SparkSession}
-
-import org.apache.carbondata.examples.util.ExampleUtils
-
-/**
- * This example is for pre-aggregate tables.
- */
-
-object PreAggregateDataMapExample {
-
- def main(args: Array[String]) {
- val spark = ExampleUtils.createCarbonSession("PreAggregateTableExample")
- exampleBody(spark)
- spark.close()
- }
-
- def exampleBody(spark : SparkSession): Unit = {
- val rootPath = new File(this.getClass.getResource("/").getPath
- + "../../../..").getCanonicalPath
- val testData = s"$rootPath/integration/spark-common-test/src/test/resources/sample.csv"
-
- // 1. simple usage for Pre-aggregate tables creation and query
- spark.sql("DROP TABLE IF EXISTS mainTable")
- spark.sql("DROP TABLE IF EXISTS mainTable_other")
- spark.sql("""
- | CREATE TABLE mainTable
- | (id Int,
- | name String,
- | city String,
- | age Int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
-
- spark.sql("""
- | CREATE TABLE mainTable_other
- | (id Int,
- | name String,
- | city String,
- | age Int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
-
- spark.sql(s"""
- LOAD DATA LOCAL INPATH '$testData' into table mainTable
- """)
-
- spark.sql(s"""
- LOAD DATA LOCAL INPATH '$testData' into table mainTable_other
- """)
-
- // 1. create pre-aggregate table by datamap
-
- // sum() be hit
- spark.sql(
- s"""create datamap preagg_sum on table mainTable using 'preaggregate' as
- | select id,sum(age) from mainTable group by id"""
- .stripMargin)
-
- // avg() and sum() both be hit, because avg = sum()/count()
- spark.sql(
- s"""create datamap preagg_avg on table mainTable using 'preaggregate' as
- | select id,avg(age) from mainTable group by id"""
- .stripMargin)
-
- // count() be hit
- spark.sql(
- s"""create datamap preagg_count_age on table mainTable using 'preaggregate' as
- | select id,count(age) from mainTable group by id"""
- .stripMargin)
-
- // min() be hit
- spark.sql(
- s"""create datamap preagg_min on table mainTable using 'preaggregate' as
- | select id,min(age) from mainTable group by id"""
- .stripMargin)
-
- // max() be hit
- spark.sql(
- s"""create datamap preagg_max on table mainTable using 'preaggregate' as
- | select id,max(age) from mainTable group by id"""
- .stripMargin)
-
- // show datamap
- spark.sql("show datamap on table mainTable").show(false)
-
- // drop datamap
- spark.sql("drop datamap preagg_count_age on table mainTable").show()
- spark.sql("show datamap on table mainTable").show(false)
-
- spark.sql(
- s"""
- | create datamap preagg_case on table mainTable using 'preaggregate' as
- | select name,sum(case when age=35 then id else 0 end) from mainTable group by name
- | """.stripMargin)
-
- spark.sql(
- s"""create datamap preagg_count on table maintable using 'preaggregate' as
- | select name, count(*) from maintable group by name""".stripMargin)
-
- spark.sql("show datamap on table maintable").show(false)
-
- spark.sql(
- s"""
- | SELECT id,max(age)
- | FROM mainTable group by id
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | select name, count(*) from
- | mainTable group by name
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | select name as NewName,
- | sum(case when age=35 then id else 0 end) as sum
- | from mainTable group by name order by name
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | select t1.name,t1.city from mainTable_other t1 join
- | (select name as newnewname,sum(age) as sum
- | from mainTable group by name) t2 on t1.name=t2.newnewname
- """.stripMargin).show()
-
- // 2.compare the performance : with pre-aggregate VS main table
-
- // build the test data, please increase the data for more obvious comparison.
- // if set the data is larger than 100M, it will take 10+ mins.
-
- import spark.implicits._
-
- import scala.util.Random
- val r = new Random()
- val df = spark.sparkContext.parallelize(1 to 10 * 10 * 1000)
- .map(x => ("No." + r.nextInt(100000), "name" + x % 8, "city" + x % 50, x % 60))
- .toDF("ID", "name", "city", "age")
-
- // Create table with pre-aggregate
- spark.sql("DROP TABLE IF EXISTS personTable")
- spark.sql("DROP TABLE IF EXISTS personTableWithoutAgg")
- df.write.format("carbondata")
- .option("tableName", "personTable")
- .option("compress", "true")
- .mode(SaveMode.Overwrite).save()
-
- // Create table without pre-aggregate
- df.write.format("carbondata")
- .option("tableName", "personTableWithoutAgg")
- .option("compress", "true")
- .mode(SaveMode.Overwrite).save()
-
- // create pre-aggregate table by datamap
- spark.sql("""
- CREATE datamap preagg_avg on table personTable using 'preaggregate' as
- | select id,avg(age) from personTable group by id
- """.stripMargin)
-
- // define time function
- def time(code: => Unit): Double = {
- val start = System.currentTimeMillis()
- code
- // return time in second
- (System.currentTimeMillis() - start).toDouble / 1000
- }
-
- val time_without_aggTable = time {
- spark.sql(
- s"""
- | SELECT id, avg(age)
- | FROM personTableWithoutAgg group by id
- """.stripMargin).count()
- }
-
- val time_with_aggTable = time {
- spark.sql(
- s"""
- | SELECT id, avg(age)
- | FROM personTable group by id
- """.stripMargin).count()
- }
- // scalastyle:off
- println("time for query on table with pre-aggregate table:" + time_with_aggTable.toString)
- println("time for query on table without pre-aggregate table:" + time_without_aggTable.toString)
- // scalastyle:on
-
- // 3. if avg function is defined for a column, sum also can be used on that;but not other way
- // round
- val time_without_aggTable_sum = time {
- spark.sql(
- s"""
- | SELECT id, sum(age)
- | FROM personTableWithoutAgg group by id
- """.stripMargin).count()
- }
-
- val time_with_aggTable_sum = time {
- spark.sql(
- s"""
- | SELECT id, sum(age)
- | FROM personTable group by id
- """.stripMargin).count()
- }
- // scalastyle:off
- println("time for query with function sum on table with pre-aggregate table:" +
- time_with_aggTable_sum.toString)
- println("time for query with function sum on table without pre-aggregate table:" +
- time_without_aggTable_sum.toString)
- // scalastyle:on
-
- spark.sql("DROP TABLE IF EXISTS mainTable")
- spark.sql("DROP TABLE IF EXISTS mainTable_other")
- spark.sql("DROP TABLE IF EXISTS personTable")
- spark.sql("DROP TABLE IF EXISTS personTableWithoutAgg")
- }
-}
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/TimeSeriesPreAggregateTableExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/TimeSeriesPreAggregateTableExample.scala
deleted file mode 100644
index 5a79e0b..0000000
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/TimeSeriesPreAggregateTableExample.scala
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import java.io.File
-
-import org.apache.spark.sql.{SaveMode, SparkSession}
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.examples.util.ExampleUtils
-
-/**
- * This example is for time series pre-aggregate tables.
- */
-
-object TimeSeriesPreAggregateTableExample {
-
- def main(args: Array[String]) {
- val spark = ExampleUtils.createCarbonSession("TimeSeriesPreAggregateTableExample")
- exampleBody(spark)
- spark.close()
- }
-
- def exampleBody(spark : SparkSession): Unit = {
- val rootPath = new File(this.getClass.getResource("/").getPath + "../../../..").getCanonicalPath
- val testData = s"$rootPath/integration/spark-common-test/src/test/resources/timeseriestest.csv"
- import spark.implicits._
-
- import scala.util.Random
- val r = new Random()
- val df = spark.sparkContext.parallelize(1 to 10 * 1000 )
- .map(x => ("" + 20 + "%02d".format(r.nextInt(20)) + "-" + "%02d".format(r.nextInt(11) + 1) +
- "-" + "%02d".format(r.nextInt(27) + 1) + " " + "%02d".format(r.nextInt(12)) + ":" +
- "%02d".format(r.nextInt(59)) + ":" + "%02d".format(r.nextInt(59)), "name" + x % 8,
- r.nextInt(60))).toDF("mytime", "name", "age")
-
- // 1. usage for time series Pre-aggregate tables creation and query
- spark.sql("drop table if exists timeSeriesTable")
- spark.sql("CREATE TABLE timeSeriesTable(mytime timestamp," +
- " name string, age int) STORED BY 'org.apache.carbondata.format'")
- spark.sql(
- s"""
- | CREATE DATAMAP agg0_hour ON TABLE timeSeriesTable
- | USING 'timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM timeSeriesTable
- | GROUP BY mytime
- """.stripMargin)
- spark.sql(
- s"""
- | CREATE DATAMAP agg0_day ON TABLE timeSeriesTable
- | USING 'timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'DAY_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM timeSeriesTable
- | GROUP BY mytime
- """.stripMargin)
-
-
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss")
-
- df.write.format("carbondata")
- .option("tableName", "timeSeriesTable")
- .option("compress", "true")
- .mode(SaveMode.Append).save()
-
- spark.sql(
- s"""
- select sum(age), timeseries(mytime,'hour') from timeSeriesTable group by timeseries(mytime,
- 'hour')
- """.stripMargin).show()
-
- spark.sql(
- s"""
- select avg(age),timeseries(mytime,'year') from timeSeriesTable group by timeseries(mytime,
- 'year')
- """.stripMargin).show()
-
- CarbonProperties.getInstance().addProperty(
- CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- CarbonProperties.getInstance().addProperty(
- CarbonCommonConstants.CARBON_DATE_FORMAT,
- CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
-
- spark.sql("DROP TABLE IF EXISTS timeSeriesTable")
- }
-}
diff --git a/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala b/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala
index cef2b97..b42fd13 100644
--- a/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala
+++ b/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala
@@ -90,10 +90,6 @@ class RunExamples extends QueryTest with BeforeAndAfterAll {
DataUpdateDeleteExample.exampleBody(spark)
}
- test("PreAggregateDataMapExample") {
- PreAggregateDataMapExample.exampleBody(spark)
- }
-
test("QuerySegmentExample") {
QuerySegmentExample.exampleBody(spark)
}
@@ -106,10 +102,6 @@ class RunExamples extends QueryTest with BeforeAndAfterAll {
TableLevelCompactionOptionExample.exampleBody(spark)
}
- test("TimeSeriesPreAggregateTableExample") {
- TimeSeriesPreAggregateTableExample.exampleBody(spark)
- }
-
test("LuceneDataMapExample") {
LuceneDataMapExample.exampleBody(spark)
}
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index 1180991..ca4bbad 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -105,6 +105,7 @@ struct ColumnSchema{
/**
* Used when this column is part of an aggregate table.
*/
+ /** Deprecated */
11: optional string aggregate_function;
12: optional binary default_value;
@@ -129,6 +130,7 @@ struct ColumnSchema{
* to maintain the column relation with parent table.
* will be usefull in case of pre-aggregate
**/
+ /** Deprecated */
17: optional list<ParentColumnTableRelation> parentColumnTableRelations;
}
@@ -204,10 +206,11 @@ struct DataMapSchema {
// stores properties of select query, query type like groupby, join in
// case of preaggregate/timeseries
3: optional map<string, string> properties;
- // relation identifier of a table which stores data of datamaps like preaggregate/timeseries.
+ // relation identifier of a table which stores data of Materialize View.
4: optional RelationIdentifier childTableIdentifier;
// in case of preaggregate/timeseries datamap it will be used to maintain the child schema
// which will be usefull in case of query and data load
+ // Deprecated
5: optional TableSchema childTableSchema;
}
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java
index 52796fb..6ab91cb 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java
@@ -110,99 +110,96 @@ public class CarbonFileInputFormat<T> extends CarbonInputFormat<T> implements Se
}
AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
- if (getValidateSegmentsToAccess(job.getConfiguration())) {
- // get all valid segments and set them into the configuration
- // check for externalTable segment (Segment_null)
- // process and resolve the expression
+ // get all valid segments and set them into the configuration
+ // check for externalTable segment (Segment_null)
+ // process and resolve the expression
- ReadCommittedScope readCommittedScope = null;
- if (carbonTable.isTransactionalTable()) {
- readCommittedScope = new LatestFilesReadCommittedScope(
- identifier.getTablePath() + "/Fact/Part0/Segment_null/", job.getConfiguration());
+ ReadCommittedScope readCommittedScope = null;
+ if (carbonTable.isTransactionalTable()) {
+ readCommittedScope = new LatestFilesReadCommittedScope(
+ identifier.getTablePath() + "/Fact/Part0/Segment_null/", job.getConfiguration());
+ } else {
+ readCommittedScope = getReadCommittedScope(job.getConfiguration());
+ if (readCommittedScope == null) {
+ readCommittedScope = new LatestFilesReadCommittedScope(identifier.getTablePath(), job
+ .getConfiguration());
} else {
- readCommittedScope = getReadCommittedScope(job.getConfiguration());
- if (readCommittedScope == null) {
- readCommittedScope = new LatestFilesReadCommittedScope(identifier.getTablePath(), job
- .getConfiguration());
- } else {
- readCommittedScope.setConfiguration(job.getConfiguration());
- }
+ readCommittedScope.setConfiguration(job.getConfiguration());
}
- // this will be null in case of corrupt schema file.
- DataMapFilter filter = getFilterPredicates(job.getConfiguration());
+ }
+ // this will be null in case of corrupt schema file.
+ DataMapFilter filter = getFilterPredicates(job.getConfiguration());
- // if external table Segments are found, add it to the List
- List<Segment> externalTableSegments = new ArrayList<Segment>();
- Segment seg;
- if (carbonTable.isTransactionalTable()) {
- // SDK some cases write into the Segment Path instead of Table Path i.e. inside
- // the "Fact/Part0/Segment_null". The segment in this case is named as "null".
- // The table is denoted by default as a transactional table and goes through
- // the path of CarbonFileInputFormat. The above scenario is handled in the below code.
- seg = new Segment("null", null, readCommittedScope);
- externalTableSegments.add(seg);
- } else {
- LoadMetadataDetails[] loadMetadataDetails = readCommittedScope.getSegmentList();
- for (LoadMetadataDetails load : loadMetadataDetails) {
- seg = new Segment(load.getLoadName(), null, readCommittedScope);
- if (fileLists != null) {
- for (int i = 0; i < fileLists.size(); i++) {
- if (fileLists.get(i).toString().endsWith(seg.getSegmentNo()
- + CarbonTablePath.CARBON_DATA_EXT)) {
- externalTableSegments.add(seg);
- break;
- }
+ // if external table Segments are found, add it to the List
+ List<Segment> externalTableSegments = new ArrayList<Segment>();
+ Segment seg;
+ if (carbonTable.isTransactionalTable()) {
+ // SDK some cases write into the Segment Path instead of Table Path i.e. inside
+ // the "Fact/Part0/Segment_null". The segment in this case is named as "null".
+ // The table is denoted by default as a transactional table and goes through
+ // the path of CarbonFileInputFormat. The above scenario is handled in the below code.
+ seg = new Segment("null", null, readCommittedScope);
+ externalTableSegments.add(seg);
+ } else {
+ LoadMetadataDetails[] loadMetadataDetails = readCommittedScope.getSegmentList();
+ for (LoadMetadataDetails load : loadMetadataDetails) {
+ seg = new Segment(load.getLoadName(), null, readCommittedScope);
+ if (fileLists != null) {
+ for (int i = 0; i < fileLists.size(); i++) {
+ if (fileLists.get(i).toString().endsWith(seg.getSegmentNo()
+ + CarbonTablePath.CARBON_DATA_EXT)) {
+ externalTableSegments.add(seg);
+ break;
}
- } else {
- externalTableSegments.add(seg);
}
+ } else {
+ externalTableSegments.add(seg);
}
}
- List<InputSplit> splits = new ArrayList<>();
- boolean useBlockDataMap = job.getConfiguration().getBoolean("filter_blocks", true);
- // useBlockDataMap would be false in case of SDK when user has not provided any filter, In
- // this case we don't want to load block/blocklet datamap. It would be true in all other
- // scenarios
- if (filter != null) {
- filter.resolve(false);
- }
- if (useBlockDataMap) {
- // do block filtering and get split
- splits = getSplits(job, filter, externalTableSegments);
+ }
+ List<InputSplit> splits = new ArrayList<>();
+ boolean useBlockDataMap = job.getConfiguration().getBoolean("filter_blocks", true);
+ // useBlockDataMap would be false in case of SDK when user has not provided any filter, In
+ // this case we don't want to load block/blocklet datamap. It would be true in all other
+ // scenarios
+ if (filter != null) {
+ filter.resolve(false);
+ }
+ if (useBlockDataMap) {
+ // do block filtering and get split
+ splits = getSplits(job, filter, externalTableSegments);
+ } else {
+ List<CarbonFile> carbonFiles = null;
+ if (null != this.fileLists) {
+ carbonFiles = getAllCarbonDataFiles(this.fileLists);
} else {
- List<CarbonFile> carbonFiles = null;
- if (null != this.fileLists) {
- carbonFiles = getAllCarbonDataFiles(this.fileLists);
- } else {
- carbonFiles = getAllCarbonDataFiles(carbonTable.getTablePath());
- }
+ carbonFiles = getAllCarbonDataFiles(carbonTable.getTablePath());
+ }
- for (CarbonFile carbonFile : carbonFiles) {
- // Segment id is set to null because SDK does not write carbondata files with respect
- // to segments. So no specific name is present for this load.
- CarbonInputSplit split =
- new CarbonInputSplit("null", carbonFile.getAbsolutePath(), 0,
- carbonFile.getLength(), carbonFile.getLocations(), FileFormat.COLUMNAR_V3);
- split.setVersion(ColumnarFormatVersion.V3);
- BlockletDetailInfo info = new BlockletDetailInfo();
- split.setDetailInfo(info);
- info.setBlockSize(carbonFile.getLength());
- info.setVersionNumber(split.getVersion().number());
- info.setUseMinMaxForPruning(false);
- splits.add(split);
- }
- Collections.sort(splits, new Comparator<InputSplit>() {
- @Override
- public int compare(InputSplit o1, InputSplit o2) {
- return ((CarbonInputSplit) o1).getFilePath()
- .compareTo(((CarbonInputSplit) o2).getFilePath());
- }
- });
+ for (CarbonFile carbonFile : carbonFiles) {
+ // Segment id is set to null because SDK does not write carbondata files with respect
+ // to segments. So no specific name is present for this load.
+ CarbonInputSplit split =
+ new CarbonInputSplit("null", carbonFile.getAbsolutePath(), 0,
+ carbonFile.getLength(), carbonFile.getLocations(), FileFormat.COLUMNAR_V3);
+ split.setVersion(ColumnarFormatVersion.V3);
+ BlockletDetailInfo info = new BlockletDetailInfo();
+ split.setDetailInfo(info);
+ info.setBlockSize(carbonFile.getLength());
+ info.setVersionNumber(split.getVersion().number());
+ info.setUseMinMaxForPruning(false);
+ splits.add(split);
}
- setAllColumnProjectionIfNotConfigured(job, carbonTable);
- return splits;
+ Collections.sort(splits, new Comparator<InputSplit>() {
+ @Override
+ public int compare(InputSplit o1, InputSplit o2) {
+ return ((CarbonInputSplit) o1).getFilePath()
+ .compareTo(((CarbonInputSplit) o2).getFilePath());
+ }
+ });
}
- return null;
+ setAllColumnProjectionIfNotConfigured(job, carbonTable);
+ return splits;
}
public void setAllColumnProjectionIfNotConfigured(JobContext job, CarbonTable carbonTable) {
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
index 5f7814a..10aabf2 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
@@ -31,7 +31,6 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.constants.CarbonCommonConstantsInternal;
import org.apache.carbondata.core.datamap.DataMapChooser;
import org.apache.carbondata.core.datamap.DataMapFilter;
import org.apache.carbondata.core.datamap.DataMapJob;
@@ -102,8 +101,6 @@ public abstract class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
// comma separated list of input segment numbers
public static final String INPUT_SEGMENT_NUMBERS =
"mapreduce.input.carboninputformat.segmentnumbers";
- private static final String VALIDATE_INPUT_SEGMENT_IDs =
- "mapreduce.input.carboninputformat.validsegments";
// comma separated list of input files
private static final String ALTER_PARTITION_ID = "mapreduce.input.carboninputformat.partitionid";
private static final Logger LOG =
@@ -321,21 +318,6 @@ m filterExpression
}
/**
- * set list of segment to access
- */
- public static void setValidateSegmentsToAccess(Configuration configuration, Boolean validate) {
- configuration.set(CarbonInputFormat.VALIDATE_INPUT_SEGMENT_IDs, validate.toString());
- }
-
- /**
- * get list of segment to access
- */
- public static boolean getValidateSegmentsToAccess(Configuration configuration) {
- return configuration.get(CarbonInputFormat.VALIDATE_INPUT_SEGMENT_IDs, "true")
- .equalsIgnoreCase("true");
- }
-
- /**
* set list of partitions to prune
*/
public static void setPartitionsToPrune(Configuration configuration,
@@ -844,25 +826,6 @@ m filterExpression
return tableName;
}
- public static void setAccessStreamingSegments(Configuration configuration, Boolean validate)
- throws InvalidConfigurationException {
- configuration.set(
- CarbonCommonConstantsInternal.QUERY_ON_PRE_AGG_STREAMING + "." + getDatabaseName(
- configuration) + "." + getTableName(configuration), validate.toString());
- }
-
- public static boolean getAccessStreamingSegments(Configuration configuration) {
- try {
- return configuration.get(
- CarbonCommonConstantsInternal.QUERY_ON_PRE_AGG_STREAMING + "." + getDatabaseName(
- configuration) + "." + getTableName(
- configuration), "false").equalsIgnoreCase("true");
-
- } catch (InvalidConfigurationException e) {
- return false;
- }
- }
-
/**
* Project all Columns for carbon reader
*
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
index 3eda548..3c14ee3 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
@@ -140,11 +140,6 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
OperationContext operationContext = (OperationContext) getOperationContext();
CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
String uuid = "";
- if (loadModel.getCarbonDataLoadSchema().getCarbonTable().isChildDataMap() &&
- operationContext != null) {
- uuid = operationContext.getProperty("uuid").toString();
- }
-
SegmentFileStore.updateTableStatusFile(carbonTable, loadModel.getSegmentId(),
segmentFileName + CarbonTablePath.SEGMENT_EXT,
carbonTable.getCarbonTableIdentifier().getTableId(),
@@ -207,8 +202,10 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
context.getConfiguration().get(CarbonTableOutputFormat.SEGMENTS_TO_BE_DELETED, "");
List<Segment> segmentDeleteList = Segment.toSegmentList(segmentsToBeDeleted.split(","), null);
Set<Segment> segmentSet = new HashSet<>(
- new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier(),
- context.getConfiguration()).getValidAndInvalidSegments(carbonTable.isChildTable())
+ new SegmentStatusManager(
+ carbonTable.getAbsoluteTableIdentifier(),
+ context.getConfiguration()
+ ).getValidAndInvalidSegments(carbonTable.isChildTableForMV())
.getValidSegments());
if (updateTime != null) {
CarbonUpdateUtil.updateTableMetadataStatus(segmentSet, carbonTable, updateTime, true,
@@ -244,7 +241,7 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
if (partitionSpecs != null && partitionSpecs.size() > 0) {
List<Segment> validSegments =
new SegmentStatusManager(table.getAbsoluteTableIdentifier())
- .getValidAndInvalidSegments(table.isChildTable()).getValidSegments();
+ .getValidAndInvalidSegments(table.isChildTableForMV()).getValidSegments();
String uniqueId = String.valueOf(System.currentTimeMillis());
List<String> tobeUpdatedSegs = new ArrayList<>();
List<String> tobeDeletedSegs = new ArrayList<>();
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 6a0f946..5468c24 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -116,54 +116,35 @@ public class CarbonTableInputFormat<T> extends CarbonInputFormat<T> {
SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier,
readCommittedScope.getConfiguration());
SegmentStatusManager.ValidAndInvalidSegmentsInfo segments = segmentStatusManager
- .getValidAndInvalidSegments(carbonTable.isChildTable(), loadMetadataDetails,
+ .getValidAndInvalidSegments(carbonTable.isChildTableForMV(), loadMetadataDetails,
this.readCommittedScope);
- // to check whether only streaming segments access is enabled or not,
- // if access streaming segment is true then data will be read from streaming segments
- boolean accessStreamingSegments = getAccessStreamingSegments(job.getConfiguration());
- if (getValidateSegmentsToAccess(job.getConfiguration())) {
- if (!accessStreamingSegments) {
- List<Segment> validSegments = segments.getValidSegments();
- streamSegments = segments.getStreamSegments();
- streamSegments = getFilteredSegment(job, streamSegments, readCommittedScope);
- if (validSegments.size() == 0) {
- return getSplitsOfStreaming(job, streamSegments, carbonTable);
- }
- List<Segment> filteredSegmentToAccess =
- getFilteredSegment(job, segments.getValidSegments(), readCommittedScope);
- if (filteredSegmentToAccess.size() == 0) {
- return getSplitsOfStreaming(job, streamSegments, carbonTable);
- } else {
- setSegmentsToAccess(job.getConfiguration(), filteredSegmentToAccess);
- }
- } else {
- List<Segment> filteredNormalSegments =
- getFilteredNormalSegments(segments.getValidSegments(),
- getSegmentsToAccess(job, readCommittedScope));
- streamSegments = segments.getStreamSegments();
- if (filteredNormalSegments.size() == 0) {
- return getSplitsOfStreaming(job, streamSegments, carbonTable);
- }
- setSegmentsToAccess(job.getConfiguration(), filteredNormalSegments);
- }
- // remove entry in the segment index if there are invalid segments
- for (Segment segment : segments.getInvalidSegments()) {
- invalidSegmentIds.add(segment.getSegmentNo());
- }
- if (invalidSegmentIds.size() > 0) {
- DataMapStoreManager.getInstance()
- .clearInvalidSegments(getOrCreateCarbonTable(job.getConfiguration()),
- invalidSegmentIds);
- }
+ List<Segment> validSegments = segments.getValidSegments();
+ streamSegments = segments.getStreamSegments();
+ streamSegments = getFilteredSegment(job, streamSegments, readCommittedScope);
+ if (validSegments.size() == 0) {
+ return getSplitsOfStreaming(job, streamSegments, carbonTable);
}
- List<Segment> validAndInProgressSegments = new ArrayList<>(segments.getValidSegments());
- // Add in progress segments also to filter it as in case of aggregate table load it loads
- // data from in progress table.
- validAndInProgressSegments.addAll(segments.getListOfInProgressSegments());
- // get updated filtered list
List<Segment> filteredSegmentToAccess =
- getFilteredSegment(job, new ArrayList<>(validAndInProgressSegments), readCommittedScope);
+ getFilteredSegment(job, segments.getValidSegments(), readCommittedScope);
+ if (filteredSegmentToAccess.size() == 0) {
+ return getSplitsOfStreaming(job, streamSegments, carbonTable);
+ } else {
+ setSegmentsToAccess(job.getConfiguration(), filteredSegmentToAccess);
+ }
+
+ // remove entry in the segment index if there are invalid segments
+ for (Segment segment : segments.getInvalidSegments()) {
+ invalidSegmentIds.add(segment.getSegmentNo());
+ }
+ if (invalidSegmentIds.size() > 0) {
+ DataMapStoreManager.getInstance()
+ .clearInvalidSegments(getOrCreateCarbonTable(job.getConfiguration()),
+ invalidSegmentIds);
+ }
+
+ List<Segment> segmentToAccess =
+ getFilteredSegment(job, segments.getValidSegments(), readCommittedScope);
// process and resolve the expression
DataMapFilter dataMapFilter = getFilterPredicates(job.getConfiguration());
@@ -174,7 +155,7 @@ public class CarbonTableInputFormat<T> extends CarbonInputFormat<T> {
// do block filtering and get split
List<InputSplit> splits = getSplits(
- job, dataMapFilter, filteredSegmentToAccess,
+ job, dataMapFilter, segmentToAccess,
updateStatusManager, segments.getInvalidSegments());
// add all splits of streaming
@@ -196,28 +177,6 @@ public class CarbonTableInputFormat<T> extends CarbonInputFormat<T> {
*/
/**
- * Below method will be used to get the filter segments when query is fired on pre Aggregate
- * and main table in case of streaming.
- * For Pre Aggregate rules it will set all the valid segments for both streaming and
- * and normal for fact table, so if any handoff happened in between it will
- * select only new hand off segments segments for fact.
- * @param validSegments
- * @param segmentsToAccess
- * @return
- */
- private List<Segment> getFilteredNormalSegments(List<Segment> validSegments,
- Segment[] segmentsToAccess) {
- List<Segment> segmentToAccessSet = Arrays.asList(segmentsToAccess);
- List<Segment> filteredSegment = new ArrayList<>();
- for (Segment seg : validSegments) {
- if (!segmentToAccessSet.contains(seg)) {
- filteredSegment.add(seg);
- }
- }
- return filteredSegment;
- }
-
- /**
* Return segment list after filtering out valid segments and segments set by user by
* `INPUT_SEGMENT_NUMBERS` in job configuration
*/
@@ -420,7 +379,7 @@ public class CarbonTableInputFormat<T> extends CarbonInputFormat<T> {
table, loadMetadataDetails);
SegmentStatusManager.ValidAndInvalidSegmentsInfo allSegments =
new SegmentStatusManager(identifier, readCommittedScope.getConfiguration())
- .getValidAndInvalidSegments(table.isChildTable(), loadMetadataDetails,
+ .getValidAndInvalidSegments(table.isChildTableForMV(), loadMetadataDetails,
readCommittedScope);
Map<String, Long> blockRowCountMapping = new HashMap<>();
Map<String, Long> segmentAndBlockCountMapping = new HashMap<>();
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 872ebb5..d78e44b 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -24,7 +24,6 @@ import java.util.Locale;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.constants.CarbonCommonConstantsInternal;
import org.apache.carbondata.core.datamap.DataMapFilter;
import org.apache.carbondata.core.datamap.DataMapJob;
import org.apache.carbondata.core.datamap.DataMapUtil;
@@ -111,7 +110,7 @@ public class CarbonInputFormatUtil {
AbsoluteTableIdentifier identifier,
DataMapFilter dataMapFilter,
CarbonProjection columnProjection,
- DataMapJob dataMapJob) throws InvalidConfigurationException, IOException {
+ DataMapJob dataMapJob) throws IOException {
CarbonTableInputFormat<V> format = new CarbonTableInputFormat<>();
CarbonInputFormat.setTablePath(
conf,
@@ -128,31 +127,12 @@ public class CarbonInputFormatUtil {
CarbonSessionInfo carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo();
if (carbonSessionInfo != null) {
String tableUniqueKey = identifier.getDatabaseName() + "." + identifier.getTableName();
- String validateInputSegmentsKey = CarbonCommonConstants.VALIDATE_CARBON_INPUT_SEGMENTS +
- tableUniqueKey;
- CarbonInputFormat.setValidateSegmentsToAccess(
- conf,
- Boolean.valueOf(carbonSessionInfo.getThreadParams().getProperty(
- validateInputSegmentsKey, "true")));
- String queryOnPreAggStreamingKey = CarbonCommonConstantsInternal.QUERY_ON_PRE_AGG_STREAMING +
- tableUniqueKey;
- boolean queryOnPreAggStreaming = Boolean.valueOf(carbonSessionInfo.getThreadParams()
- .getProperty(queryOnPreAggStreamingKey, "false"));
String inputSegmentsKey = CarbonCommonConstants.CARBON_INPUT_SEGMENTS + tableUniqueKey;
- CarbonInputFormat.setValidateSegmentsToAccess(conf,
- Boolean.valueOf(carbonSessionInfo.getThreadParams()
- .getProperty(validateInputSegmentsKey, "true")));
CarbonInputFormat.setQuerySegment(
conf,
carbonSessionInfo.getThreadParams().getProperty(
inputSegmentsKey,
CarbonProperties.getInstance().getProperty(inputSegmentsKey, "*")));
- if (queryOnPreAggStreaming) {
- CarbonInputFormat.setAccessStreamingSegments(conf, true);
- carbonSessionInfo.getThreadParams().removeProperty(queryOnPreAggStreamingKey);
- carbonSessionInfo.getThreadParams().removeProperty(inputSegmentsKey);
- carbonSessionInfo.getThreadParams().removeProperty(validateInputSegmentsKey);
- }
}
return format;
}
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/PartitionWithPreAggregateTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/PartitionWithPreAggregateTestCase.scala
deleted file mode 100644
index 23d5924..0000000
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/PartitionWithPreAggregateTestCase.scala
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.cluster.sdv.generated
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.common.util._
-import org.scalatest.BeforeAndAfterAll
-/**
- * Test Class for Support of Partition with PreAggregate table
- */
-class PartitionWithPreAggregateTestCase extends QueryTest with BeforeAndAfterAll {
- override def beforeAll = {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
- .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
- }
-
- //Loading data into partitioned table with SORT_SCOPE=LOCAL_SORT
- test("Partition-With-PreAggregate_TC001", Include) {
- sql("drop table if exists partition_table")
- sql(
- s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG,
- |doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE,
- |charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY
- |'carbondata' TBLPROPERTIES('SORT_SCOPE'='LOCAL_SORT')""".stripMargin)
- sql(
- s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table
- |partition_table""".stripMargin)
- sql(
- "create datamap ag1 on table partition_table using 'preaggregate' as select shortField, sum" +
- "(intField) from partition_table group by shortField")
- checkAnswer(sql(
- s"""select decimalfield from partition_table where charfield='e' and
- |floatfield=307.301 group by decimalfield limit 1""".stripMargin),
- Seq(Row(159.10)))
- }
-
- //Loading data into partitioned table with SORT_SCOPE=GLOBAL_SORT
- test("Partition-With-PreAggregate_TC002", Include) {
- sql("drop table if exists partition_table")
- sql(
- s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG,
- |doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE,
- |charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY
- |'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')""".stripMargin)
- sql(
- s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table
- |partition_table""".stripMargin)
- sql(
- "create datamap ag1 on table partition_table using 'preaggregate' as select shortField, sum" +
- "(intField) from partition_table group by shortField")
- checkAnswer(sql(
- s"""select decimalfield from partition_table where charfield='e' and
- |floatfield=307.301 group by decimalfield limit 1""".stripMargin),
- Seq(Row(159.10)))
- }
-
- //Loading data into partitioned table with SORT_SCOPE=NO_SORT
- test("Partition-With-PreAggregate_TC004", Include) {
- sql("drop table if exists partition_table")
- sql(
- s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG,
- |doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE,
- |charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY
- |'carbondata' TBLPROPERTIES('SORT_SCOPE'='NO_SORT')""".stripMargin)
- sql(
- s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table
- |partition_table""".stripMargin)
- sql(
- "create datamap ag1 on table partition_table using 'preaggregate' as select shortField, sum" +
- "(intField) from partition_table group by shortField")
- checkAnswer(sql(
- s"""select decimalfield from partition_table where charfield='e' and
- |floatfield=307.301 group by decimalfield limit 1""".stripMargin),
- Seq(Row(159.10)))
- }
-
- //Verify Aggregation query on Loaded Partition table with Pre-Aggregate table
- test("Partition-With-PreAggregate_TC005", Include) {
- sql("drop table if exists partition_table")
- sql(
- s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG,
- |doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE,
- |charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY
- |'carbondata'""".stripMargin)
- sql(
- "create datamap ag1 on table partition_table using 'preaggregate' as select shortField, sum" +
- "(intField) from partition_table group by shortField")
- sql(
- s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table
- |partition_table""".stripMargin)
- assert(sql("explain select shortField, sum(intField) from partition_table group by shortField")
- .collect().head.get(0).toString.contains("partition_table_ag1"))
- }
-
- //Verify load with Pre_Aggregate table on Partition table after compaction
- test("Partition-With-PreAggregate_TC006", Include) {
- sql("drop table if exists uniqdata")
- sql(
- s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp,
- |BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),
- |DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,
- |INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY
- |'carbondata'""".stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql("create datamap ag1 on table uniqdata using 'preaggregate' as select cust_id, " +
- "sum(Double_COLUMN1) from uniqdata group by cust_id")
- sql(s"""alter table uniqdata compact 'minor'""")
- assert(sql("show segments for table uniqdata").collect().tail(2).get(0).toString.equals("0.1"))
- }
-
- //Verify load with Pre_Aggregate table on Partition table after two-level compaction
- test("Partition-With-PreAggregate_TC007", Include) {
- sql("drop table if exists uniqdata")
- sql(
- s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp,
- |BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),
- |DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,
- |INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY
- |'carbondata'""".stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql("create datamap ag1 on table uniqdata using 'preaggregate' as select cust_id, " +
- "sum(Double_COLUMN1) from uniqdata group by cust_id")
- sql(s"""alter table uniqdata compact 'minor'""")
- sql(s"""alter table uniqdata compact 'major'""")
- assert(sql("show segments for table uniqdata").collect().tail(7).get(0).toString.equals("0.2"))
- }
-
- //Verify join operation on Partitonwith Pre-Aggregate table
- test("Partition-With-PreAggregate_TC008", Include) {
- sql("drop table if exists uniqdata")
- sql(s"""drop table if exists uniqdata1""")
- sql(
- s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp,
- |BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),
- |DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,
- |INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'carbondata'"""
- .stripMargin)
- sql(
- s"""CREATE TABLE uniqdata1 (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp,
- |BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),
- |DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,
- |INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'carbondata'"""
- .stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata1 partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- sql("create datamap ag1 on table uniqdata using 'preaggregate' as select cust_id, " +
- "sum(Double_COLUMN1) from uniqdata group by cust_id")
- sql("create datamap ag1 on table uniqdata1 using 'preaggregate' as select cust_id, " +
- "sum(Double_COLUMN1) from uniqdata1 group by cust_id")
- checkAnswer(sql(
- s""" select a.cust_id, b.cust_id from uniqdata a, uniqdata1 b where
- |a.cust_id >= b.cust_id limit 1""".stripMargin),Seq(Row(1,1)))
- sql(s"""drop table if exists uniqdata1""")
- }
-
- //Verify date with > filter condition on Partitonwith Pre-Aggregate table
- test("Partition-With-PreAggregate_TC009", Include) {
- sql("drop table if exists uniqdata")
- sql(
- s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp,
- |BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),
- |DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,
- |INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'carbondata'"""
- .stripMargin)
- sql("create datamap ag1 on table uniqdata using 'preaggregate' as select cust_id, " +
- "sum(Double_COLUMN1) from uniqdata group by cust_id")
- sql(
- s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table
- | uniqdata partition(CUST_ID='4') OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE',
- | 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""".stripMargin)
- checkAnswer(sql(
- s"""select cust_name from uniqdata where active_emui_version =
- |'ACTIVE_EMUI_VERSION_00000'""".stripMargin),Seq(Row("CUST_NAME_00000")))
- }
-
- override def afterAll {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
- CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
- sql("drop table if exists uniqdata")
- sql("drop table if exists partition_table")
- sql(s"""drop table if exists uniqdata1""")
-
- }
-}
\ No newline at end of file
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/PreAggregateTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/PreAggregateTestCase.scala
deleted file mode 100644
index c949be6..0000000
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/PreAggregateTestCase.scala
+++ /dev/null
@@ -1,230 +0,0 @@
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.cluster.sdv.generated
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.common.util.{Include, QueryTest}
-import org.junit.Assert
-import org.scalatest.BeforeAndAfterEach
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-
-class PreAggregateTestCase extends QueryTest with BeforeAndAfterEach {
- val csvPath = s"$resourcesPath/source.csv"
-
- override def beforeEach: Unit = {
- sql("drop table if exists PreAggMain")
- sql("drop table if exists AggMain")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
- sql("CREATE TABLE PreAggMain (id Int, date date, country string, phonetype string, " +
- "serialname String,salary int ) STORED BY 'org.apache.carbondata.format' " +
- "tblproperties('dictionary_include'='country')")
- sql("CREATE TABLE AggMain (id Int, date date, country string, phonetype string, " +
- "serialname String,salary int ) STORED BY 'org.apache.carbondata.format'" +
- "tblproperties('dictionary_include'='country')")
- sql("create datamap PreAggSum on table PreAggMain using 'preaggregate' as " +
- "select country,sum(salary) as sum from PreAggMain group by country")
- sql("create datamap PreAggAvg on table PreAggMain using 'preaggregate' as " +
- "select country,avg(salary) as avg from PreAggMain group by country")
- sql("create datamap PreAggCount on table PreAggMain using 'preaggregate' as " +
- "select country,count(salary) as count from PreAggMain group by country")
- sql("create datamap PreAggMin on table PreAggMain using 'preaggregate' as " +
- "select country,min(salary) as min from PreAggMain group by country")
- sql("create datamap PreAggMax on table PreAggMain using 'preaggregate' as " +
- "select country,max(salary) as max from PreAggMain group by country")
- }
-
- //test to check existence of datamap
- test("PreAggregateTestCase_TC001", Include) {
- Assert.assertEquals(sql("show datamap on table PreAggMain").count(), 5)
- checkExistence(sql("Describe formatted PreAggMain_PreAggSum"), true, "Dictionary")
- }
-
- //check for load data should reflects in all preaggregate tables
- test("PreAggregateTestCase_TC002", Include) {
- sql(s"LOAD DATA INPATH '$csvPath' into table PreAggMain ").collect
- sql(s"LOAD DATA INPATH '$csvPath' into table AggMain ").collect
-
- val expectedSum = sql("select country,sum(salary) as sum from AggMain group by country")
- checkAnswer(sql("select * from PreAggMain_PreAggSum"), expectedSum)
-
- val expectedAvg = sql("select country,sum(salary),count(salary) from AggMain group by country")
- checkAnswer(sql("select * from PreAggMain_PreAggAvg"), expectedAvg)
-
- val expectedCount = sql("select country,count(salary) as count from AggMain group by country")
- checkAnswer(sql("select * from PreAggMain_PreAggCount"), expectedCount)
-
- val expectedMin = sql("select country,min(salary) as min from AggMain group by country")
- checkAnswer(sql("select * from PreAggMain_PreAggMin"), expectedMin)
-
- val expectedMax = sql("select country,max(salary) as max from AggMain group by country")
- checkAnswer(sql("select * from PreAggMain_PreAggMax"), expectedMax)
- }
-
-
- //test for incremental load
- test("PreAggregateTestCase_TC003", Include) {
- sql(s"LOAD DATA INPATH '$csvPath' into table PreAggMain").collect
- sql(s"LOAD DATA INPATH '$csvPath' into table PreAggMain").collect
- sql(s"LOAD DATA INPATH '$csvPath' into table AggMain").collect
-
- val expectedSum = sql("select country,sum(salary) as sum from " +
- "AggMain group by country")
- checkAnswer(sql("select * from PreAggMain_PreAggSum"), expectedSum.union(expectedSum))
-
- val expectedAvg = sql("select country,sum(salary),count(salary) from AggMain group by country")
- checkAnswer(sql("select * from PreAggMain_PreAggAvg"), expectedAvg.union(expectedAvg))
-
- val expectedCount = sql("select country,count(salary) as count from AggMain group by country")
- checkAnswer(sql("select * from PreAggMain_PreAggCount"), expectedCount.union(expectedCount))
-
- val expectedMin = sql("select country,min(salary) as min from AggMain group by country")
- checkAnswer(sql("select * from PreAggMain_PreAggMin"), expectedMin.union(expectedMin))
-
- val expectedMax = sql("select country,max(salary) as max from AggMain group by country")
- checkAnswer(sql("select * from PreAggMain_PreAggMax"), expectedMax.union(expectedMax))
- }
-
- //test for creating datamap having data from all segment after incremental load
- test("PreAggregateTestCase_TC004", Include) {
- sql("insert into PreAggMain values(1,'2015/7/23','country1','phone197','ASD69643',15000)")
- sql("insert into PreAggMain values(2,'2015/8/23','country2','phone197','ASD69643',10000)")
- sql("insert into PreAggMain values(3,'2005/7/28','country1','phone197','ASD69643',5000)")
- sql("create datamap testDataMap on table PreAggMain using 'preaggregate' as " +
- "select country,sum(salary) as sum from PreAggMain group by country")
- checkAnswer(sql("select * from PreAggMain_testDataMap"),
- Seq(Row("country1", 20000), Row("country2", 10000)))
- }
-
- //test for insert overwrite in main table
- test("PreAggregateTestCase_TC005", Include) {
- sql("insert into PreAggMain values(1,'2015/7/23','country1','phone197','ASD696',15000)")
- sql("insert into PreAggMain values(2,'2003/8/13','country2','phone197','AD6943',10000)")
- sql("insert overwrite table PreAggMain values(3,'2005/7/28','country3','phone197','ASD69643',5000)")
- sql("create datamap testDataMap on table PreAggMain using 'preaggregate' as " +
- "select country,sum(salary) as sum from PreAggMain group by country")
- checkAnswer(sql("select * from PreAggMain_testDataMap"), Seq(Row("country3", 5000)))
- }
-
- // test output for join query with preaggregate and without preaggregate
- test("PreAggregateTestCase_TC006", Include) {
- sql(s"LOAD DATA INPATH '$csvPath' into table PreAggMain").collect
- sql(s"LOAD DATA INPATH '$csvPath' into table AggMain").collect
- val actual = sql("select t1.country,sum(id) from PreAggMain t1 join " +
- "(select country as newcountry,sum(salary) as sum from PreAggMain group by " +
- "country )t2 on t1.country=t2.newcountry group by country")
-
- val expected = sql("select t1.country,sum(id) from AggMain t1 join " +
- "(select country as newcountry,sum(salary) as sum from AggMain group by " +
- "country )t2 on t1.country=t2.newcountry group by country")
- checkAnswer(actual, expected)
- }
-
- test("PreAggregateTestCase_TC007", Include) {
- sql(s"LOAD DATA INPATH '$csvPath' into table PreAggMain").collect
- sql(s"LOAD DATA INPATH '$csvPath' into table AggMain").collect
- val actual = sql("select t1.country,count(t1.country) from PreAggMain t1 join " +
- " (select country,count(salary) as count from PreAggMain group by country )" +
- "t2 on t1.country=t2.country group by t1.country")
-
- val expected = sql("select t1.country,count(t1.country) from AggMain t1 join " +
- "(select country,count(salary) as count from AggMain group by country )t2 " +
- "on t1.country=t2.country group by t1.country")
- checkAnswer(actual, expected)
- }
-
- //test to verify correct data in preaggregate table
- test("PreAggregateTestCase_TC008", Include) {
- sql(s"LOAD DATA INPATH '$csvPath' into table PreAggMain").collect
- sql(s"LOAD DATA INPATH '$csvPath' into table AggMain").collect
- sql("create datamap testDatamap on table PreAggMain using 'preaggregate' as " +
- "select sum(CASE WHEN country='china' THEN id ELSE 0 END) as sum,country from " +
- "PreAggMain group by country")
- val actual = sql("select * from PreAggMain_testDatamap")
- val expected = sql(
- "select sum(CASE WHEN country='china' THEN id ELSE 0 END) as sum,country " +
- "from AggMain group by country")
- checkAnswer(actual, expected)
- }
-
- //test for select using in clause in preaggregate table
- test("PreAggregateTestCase_TC009", Include) {
- sql(s"LOAD DATA INPATH '$csvPath' into table PreAggMain").collect
- sql(s"LOAD DATA INPATH '$csvPath' into table AggMain").collect
- sql("create datamap testDatamap on table PreAggMain using 'preaggregate' as " +
- "select sum(CASE WHEN id in (10,11,12) THEN salary ELSE 0 END) as sum from PreAggMain " +
- "group by country")
- val actual = sql("select * from PreAggMain_testDatamap")
- val expected = sql(
- "select sum(CASE WHEN id in (10,11,12) THEN salary ELSE 0 END) as sum,country from AggMain " +
- "group by country")
- checkAnswer(actual, expected)
- }
-
- //test to check data using preaggregate and without preaggregate
- test("PreAggregateTestCase_TC010", Include) {
- sql(s"LOAD DATA INPATH '$csvPath' into table PreAggMain").collect
- sql(s"LOAD DATA INPATH '$csvPath' into table AggMain").collect
-
- val actual = sql(
- "select count(CASE WHEN country='usa' THEN id ELSE 0 END) as count,country from PreAggMain " +
- "group by country")
- val expected = sql(
- "select count(CASE WHEN country='usa' THEN id ELSE 0 END) as count,country from AggMain group" +
- " by country")
- checkAnswer(actual, expected)
- }
-
- //test to check data using preaggregate and without preaggregate with in clause
- test("PreAggregateTestCase_TC011", Include) {
- sql(s"LOAD DATA INPATH '$csvPath' into table PreAggMain").collect
- sql(s"LOAD DATA INPATH '$csvPath' into table AggMain").collect
- sql("create datamap testDatamap on table PreAggMain using 'preaggregate' as " +
- "select sum(CASE WHEN id in (12,13,14) THEN salary ELSE 0 END) as sum from PreAggMain " +
- "group by country")
- val actual = sql(
- "select sum(CASE WHEN id in (12,13,14) THEN salary ELSE 0 END) as sum,country from " +
- "PreAggMain group by country")
- val expected = sql(
- "select sum(CASE WHEN id in (12,13,14) THEN salary ELSE 0 END) as sum,country from AggMain " +
- "group by country")
- checkAnswer(actual, expected)
- }
-
- test("Test CleanUp of Pre_aggregate tables") {
- sql("drop table if exists maintable")
- sql("create table maintable(name string, c_code int, price int) stored by 'carbondata'")
- sql("insert into table maintable select 'abc',21,2000")
- sql("create datamap ag1 on table maintable using 'preaggregate' as select name,sum(price) from maintable group by name")
- sql("insert into table maintable select 'abcd',22,3000")
- sql("insert into table maintable select 'abcd',22,3000")
- sql("insert into table maintable select 'abcd',22,3000")
- sql("alter table maintable compact 'minor'")
- sql("clean files for table maintable")
- assert(sql("show segments for table maintable").collect().head.get(0).toString.contains("0.1"))
- }
-
- override def afterEach: Unit = {
- sql("drop table if exists mainTable")
- }
-
-}
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/TimeSeriesPreAggregateTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/TimeSeriesPreAggregateTestCase.scala
deleted file mode 100644
index ab05a84..0000000
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/TimeSeriesPreAggregateTestCase.scala
+++ /dev/null
@@ -1,192 +0,0 @@
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.cluster.sdv.generated
-
-import java.util.TimeZone
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-import org.scalatest.Matchers._
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.TIMESERIES
-import org.apache.carbondata.core.util.CarbonProperties
-
-class TimeSeriesPreAggregateTestCase extends QueryTest with BeforeAndAfterAll {
-
- val timeSeries = TIMESERIES.toString
- val timeZonePre = TimeZone.getDefault
- val csvPath = s"$resourcesPath/Data/timeseriestest.csv"
- override def beforeAll: Unit = {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- TimeZone.setDefault(TimeZone.getTimeZone(System.getProperty("user.timezone")))
- sql("drop table if exists mainTable")
- sql(
- "CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache" +
- ".carbondata.format'")
- sql(
- s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'SECOND_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_minute ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'minute_granularity'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_hour ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_day ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'DAY_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_month ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'MONTH_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_year ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'year_granularity'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- sql(s"LOAD DATA LOCAL INPATH '$csvPath' into table mainTable")
- }
-
- test("TimeSeriesPreAggregateTestCase_001") {
- val expected = sql("select cast(date_format(mytime, 'YYYY') as timestamp) as mytime,sum(age) " +
- "from mainTable group by date_format(mytime , 'YYYY')")
- val actual = sql("select * from maintable_agg0_year")
- checkAnswer(actual, expected)
- }
-
- test("TimeSeriesPreAggregateTestCase_002") {
- val expected = sql(
- "select cast(date_format(mytime, 'YYYY-MM') as timestamp) as mytime,sum(age) " +
- "from mainTable group by date_format(mytime , 'YYYY-MM')")
- val actual = sql("select * from maintable_agg0_month")
- checkAnswer(actual, expected)
- }
-
- test("TimeSeriesPreAggregateTestCase_003") {
- val expected = sql(
- "select cast(date_format(mytime, 'YYYY-MM-dd') as timestamp) as mytime,sum(age) " +
- "from mainTable group by date_format(mytime , 'YYYY-MM-dd')")
- val actual = sql("select * from maintable_agg0_day")
- checkAnswer(actual, expected)
- }
-
- test("TimeSeriesPreAggregateTestCase_004") {
- val expected = sql(
- "select cast(date_format(mytime, 'YYYY-MM-dd HH') as timestamp) as mytime,sum(age) " +
- "from mainTable group by date_format(mytime , 'YYYY-MM-dd HH')")
- val actual = sql("select * from maintable_agg0_hour")
- checkAnswer(actual, expected)
- }
-
- test("TimeSeriesPreAggregateTestCase_005") {
- val expected = sql(
- "select cast(date_format(mytime, 'YYYY-MM-dd HH:mm') as timestamp) as mytime,sum(age) " +
- "from mainTable group by date_format(mytime , 'YYYY-MM-dd HH:mm')")
- val actual = sql("select * from maintable_agg0_minute")
- checkAnswer(actual, expected)
- }
-
- test("TimeSeriesPreAggregateTestCase_006") {
- val expected = sql(
- "select cast(date_format(mytime, 'YYYY-MM-dd HH:mm:ss') as timestamp) as mytime,sum(age) " +
- "from mainTable group by date_format(mytime , 'YYYY-MM-dd HH:mm:ss')")
- val actual = sql("select * from maintable_agg0_second")
- checkAnswer(actual, expected)
- }
-
- //test case for compaction
- test("TimeSeriesPreAggregateTestCase_007") {
- sql(s"LOAD DATA LOCAL INPATH '$csvPath' into table mainTable")
- sql(s"LOAD DATA LOCAL INPATH '$csvPath' into table mainTable")
- sql(s"LOAD DATA LOCAL INPATH '$csvPath' into table mainTable")
- sql("alter table maintable compact 'minor'")
- val segmentNamesSecond = sql("show segments for table maintable_agg0_second").collect()
- .map(_.get(0).toString)
- segmentNamesSecond should equal(Array("3", "2", "1", "0.1", "0"))
-
- val segmentNamesMinute = sql("show segments for table maintable_agg0_minute").collect()
- .map(_.get(0).toString)
- segmentNamesMinute should equal(Array("3", "2", "1", "0.1", "0"))
-
- val segmentNamesHour = sql("show segments for table maintable_agg0_hour").collect()
- .map(_.get(0).toString)
- segmentNamesHour should equal(Array("3", "2", "1", "0.1", "0"))
-
- val segmentNamesday = sql("show segments for table maintable_agg0_day").collect()
- .map(_.get(0).toString)
- segmentNamesday should equal(Array("3", "2", "1", "0.1", "0"))
-
- val segmentNamesmonth = sql("show segments for table maintable_agg0_month").collect()
- .map(_.get(0).toString)
- segmentNamesmonth should equal(Array("3", "2", "1", "0.1", "0"))
-
- val segmentNamesyear = sql("show segments for table maintable_agg0_year").collect()
- .map(_.get(0).toString)
- segmentNamesyear should equal(Array("3", "2", "1", "0.1", "0"))
- }
-
- override def afterAll: Unit = {
- TimeZone.setDefault(timeZonePre)
- sql("drop table if exists mainTable")
-
- }
-}
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/register/TestRegisterCarbonTable.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/register/TestRegisterCarbonTable.scala
index caae8e1..7d93f15 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/register/TestRegisterCarbonTable.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/register/TestRegisterCarbonTable.scala
@@ -109,73 +109,6 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("select c1 from carbontable"), Seq(Row("a")))
}
- test("register pre aggregate tables test") {
- sql("drop database if exists carbon cascade")
- sql(s"create database carbon location '$dbLocationCustom'")
- sql("use carbon")
- sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
- sql("insert into carbontable select 'a',1,'aa','aaa'")
- sql("insert into carbontable select 'b',1,'aa','aaa'")
- sql("insert into carbontable select 'a',10,'aa','aaa'")
- sql("create datamap preagg1 on table carbontable using 'preaggregate' as select c1,sum(c2) from carbontable group by c1")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
- backUpData(dbLocationCustom, "carbontable")
- backUpData(dbLocationCustom, "carbontable_preagg1")
- sql("drop table carbontable")
- restoreData(dbLocationCustom, "carbontable")
- restoreData(dbLocationCustom, "carbontable_preagg1")
- sql("refresh table carbontable")
- }
- checkAnswer(sql("select count(*) from carbontable"), Row(3))
- checkAnswer(sql("select c1 from carbontable"), Seq(Row("a"), Row("b"), Row("a")))
- checkAnswer(sql("select count(*) from carbontable_preagg1"), Row(2))
- checkAnswer(sql("select carbontable_c1 from carbontable_preagg1"), Seq(Row("a"), Row("b")))
- }
-
- test("register pre aggregate table test") {
- sql("drop database if exists carbon cascade")
- sql(s"create database carbon location '$dbLocationCustom'")
- sql("use carbon")
- sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
- sql("insert into carbontable select 'a',1,'aa','aaa'")
- sql("insert into carbontable select 'b',1,'aa','aaa'")
- sql("insert into carbontable select 'a',10,'aa','aaa'")
- sql("create datamap preagg1 on table carbontable using 'preaggregate' as select c1,sum(c2) from carbontable group by c1")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
- backUpData(dbLocationCustom, "carbontable")
- backUpData(dbLocationCustom, "carbontable_preagg1")
- sql("drop table carbontable")
- restoreData(dbLocationCustom, "carbontable")
- restoreData(dbLocationCustom, "carbontable_preagg1")
- sql("refresh table carbontable")
- }
- checkAnswer(sql("select count(*) from carbontable"), Row(3))
- checkAnswer(sql("select c1 from carbontable"), Seq(Row("a"), Row("b"), Row("a")))
- checkAnswer(sql("select count(*) from carbontable_preagg1"), Row(2))
- checkAnswer(sql("select carbontable_c1 from carbontable_preagg1"), Seq(Row("a"), Row("b")))
- }
-
- test("register pre aggregate table should fail if the aggregate table not copied") {
- sql("drop database if exists carbon cascade")
- sql(s"create database carbon location '$dbLocationCustom'")
- sql("use carbon")
- sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
- sql("insert into carbontable select 'a',1,'aa','aaa'")
- sql("insert into carbontable select 'b',1,'aa','aaa'")
- sql("insert into carbontable select 'a',10,'aa','aaa'")
- sql("create datamap preagg1 on table carbontable using 'preaggregate' as select c1,sum(c2) from carbontable group by c1")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
- backUpData(dbLocationCustom, "carbontable")
- backUpData(dbLocationCustom, "carbontable_preagg1")
- sql("drop table carbontable")
- restoreData(dbLocationCustom, "carbontable")
- intercept[ProcessMetaDataException] {
- sql("refresh table carbontable")
- }
- restoreData(dbLocationCustom, "carbontable_preagg1")
- }
- }
-
test("Update operation on carbon table should pass after registration or refresh") {
sql("drop database if exists carbon cascade")
sql(s"create database carbon location '$dbLocationCustom'")
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
index dd32356..1d14d06 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
@@ -57,10 +57,7 @@ class SDVSuites extends Suites with BeforeAndAfterAll {
new StandardPartitionTestCase ::
new Vector1TestCase ::
new Vector2TestCase ::
- new PreAggregateTestCase ::
- new TimeSeriesPreAggregateTestCase ::
new TestPartitionWithGlobalSort ::
- new PartitionWithPreAggregateTestCase ::
new CreateTableWithLocalDictionaryTestCase ::
new LoadTableWithLocalDictionaryTestCase :: Nil
@@ -157,13 +154,10 @@ class SDVSuites4 extends Suites with BeforeAndAfterAll {
new TestRegisterCarbonTable ::
new TableCommentAlterTableTestCase ::
new StandardPartitionTestCase ::
- new PreAggregateTestCase ::
new LuceneTestCase ::
- new TimeSeriesPreAggregateTestCase ::
new TestPartitionWithGlobalSort ::
new SDKwriterTestCase ::
new SetParameterTestCase ::
- new PartitionWithPreAggregateTestCase ::
new CreateTableWithLocalDictionaryTestCase ::
new LoadTableWithLocalDictionaryTestCase :: Nil
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
index 9c0ab89..09a93c3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -471,24 +471,6 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
sql("drop datamap if exists dm on table datamap_test_table")
}
- test("test lucene fine grain data map for show datamaps with Preaggregate and Lucene") {
- sql("DROP TABLE IF EXISTS datamap_main")
- sql("create table datamap_main (a string, b string, c string) stored by 'carbondata'")
- sql(
- s"""
- | CREATE DATAMAP dm_lucene ON TABLE datamap_main
- | USING 'lucene'
- | DMProperties('INDEX_COLUMNS'='c')
- """.stripMargin)
- sql(
- "create datamap dm_pre on table datamap_main USING 'preaggregate' as select a,sum(b) " +
- "from datamap_main group by a")
- checkExistence(sql("show datamap on table datamap_main"), true, "dm_pre")
- checkExistence(sql("show datamap on table datamap_main"), true, "dm_lucene")
- sql("drop datamap if exists dm_pre on table datamap_main")
- sql("drop datamap if exists dm_lucene on table datamap_main")
- }
-
test("test lucene fine grain data map with CTAS") {
sql("DROP TABLE IF EXISTS source_table")
sql("DROP TABLE IF EXISTS target_table")
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
index 4fe0c46..bfc1935 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
@@ -827,23 +827,6 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
assertResult("Cannot use struct<b:int> for partition column;")(structException.getMessage)
}
- test("test block preaggregate") {
- sql("DROP TABLE IF EXISTS test")
- sql("create table test(id int,a struct<b:int>) stored by 'carbondata'")
- sql("insert into test values(1, named_struct('b', 2))")
- sql("insert into test values(1, named_struct('b', 2))")
- sql("insert into test values(1, named_struct('b', 2))")
- val structException = intercept[UnsupportedOperationException](
- sql("create datamap preagg_sum on table test using 'preaggregate' as select id,sum(a.b) from test group by id"))
- assertResult("Preaggregate is unsupported for ComplexData type column: a.b")(structException.getMessage)
- sql("DROP TABLE IF EXISTS test")
- sql("create table test(id int,a array<int>) stored by 'carbondata'")
- sql("insert into test values(1, array(2))")
- val arrayException = intercept[UnsupportedOperationException](
- sql("create datamap preagg_sum on table test using 'preaggregate' as select id,sum(a[0]) from test group by id"))
- assertResult("Preaggregate is unsupported for ComplexData type column: a[0]")(arrayException.getMessage)
- }
-
test("test block dictionary exclude for child column") {
sql("DROP TABLE IF EXISTS table1")
sql(
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
deleted file mode 100644
index 5aecb08..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ /dev/null
@@ -1,534 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.integration.spark.testsuite.preaggregate
-
-import java.io.File
-import java.util
-import java.util.concurrent.{Callable, ExecutorService, Executors, TimeUnit}
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.{AnalysisException, CarbonDatasourceHadoopRelation, Row}
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.sql.hive.CarbonRelation
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, MalformedDataMapCommandException}
-import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.TIMESERIES
-import org.apache.carbondata.core.util.CarbonProperties
-
-class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
-
- val timeSeries = TIMESERIES.toString
-
- override def beforeAll {
- sql("drop database if exists otherDB cascade")
- sql("drop table if exists PreAggMain")
- sql("drop table if exists PreAggMain1")
- sql("drop table if exists maintable")
- sql("drop table if exists showTables")
- sql("drop table if exists Preagg_twodb")
- sql("create table preaggMain (a string, b string, c string) stored by 'carbondata'")
- sql("create table preaggMain1 (a string, b string, c string) stored by 'carbondata' tblProperties('DICTIONARY_INCLUDE' = 'a')")
- sql("create table maintable (column1 int, column6 string, column5 string, column2 string, column3 int, column4 int) stored by 'carbondata' tblproperties('dictionary_include'='column1,column6', 'dictionary_exclude'='column3,column5')")
- }
-
- test("test pre agg create table 1") {
- sql("create datamap preagg1 on table PreAggMain using 'preaggregate' as select a,sum(b) from PreAggMain group by a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg1"), true, "preaggmain_a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg1"), true, "preaggmain_b_sum")
- sql("drop datamap preagg1 on table PreAggMain")
- }
-
- test("test pre agg create table 2") {
- dropDataMaps("PreAggMain", "preagg2")
- sql("create datamap preagg2 on table PreAggMain using 'preaggregate' as select a as a1,sum(b) as udfsum from PreAggMain group by a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg2"), true, "preaggmain_a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg2"), true, "preaggmain_b_sum")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg2"), false, "preaggmain_a1")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg2"), false, "preaggmain_udfsum")
- sql("drop datamap preagg2 on table PreAggMain")
- }
-
- test("test pre agg create table 5") {
- sql("create datamap preagg11 on table PreAggMain1 using 'preaggregate'as select a,sum(b) from PreAggMain1 group by a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain1_preagg11"), true, "preaggmain1_a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain1_preagg11"), true, "preaggmain1_b_sum")
- sql("DESCRIBE FORMATTED PreAggMain1_preagg11").show(100, false)
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain1_preagg11"), true, "Dictionary")
- sql("drop datamap preagg11 on table PreAggMain1")
- }
-
- test("test pre agg create table 6") {
- sql("create datamap preagg12 on table PreAggMain1 using 'preaggregate' as select a as a1,sum(b) as sum from PreAggMain1 group by a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain1_preagg12"), true, "preaggmain1_a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain1_preagg12"), true, "preaggmain1_b_sum")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain1_preagg12"), false, "preaggmain1_a1")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain1_preagg12"), false, "preaggmain1_sum")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain1_preagg12"), true, "Dictionary")
- sql("drop datamap preagg12 on table PreAggMain1")
- }
-
- test("test pre agg create table 8") {
- sql("create datamap preagg14 on table PreAggMain1 using 'preaggregate' as select a as a1,sum(b) as sum from PreAggMain1 group by a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain1_preagg14"), true, "preaggmain1_a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain1_preagg14"), true, "preaggmain1_b_sum")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain1_preagg14"), false, "preaggmain1_a1")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain1_preagg14"), false, "preaggmain1_sum")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain1_preagg14"), true, "Dictionary")
- sql("drop datamap preagg14 on table PreAggMain1")
- }
-
- test("test pre agg create table 9") {
- sql("create datamap preagg15 on table PreAggMain using 'preaggregate' as select a,avg(b) from PreAggMain group by a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg15"), true, "preaggmain_a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg15"), true, "preaggmain_b_sum")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg15"), true, "preaggmain_b_count")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg15"), false, "preaggmain2_b_avg")
- sql("drop datamap preagg15 on table PreAggMain")
- }
-
- test("test pre agg create table 10") {
- sql("create datamap preagg16 on table PreAggMain using 'preaggregate' as select a as a1,max(b) from PreAggMain group by a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg16"), true, "preaggmain_a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg16"), true, "preaggmain_b_max")
- sql("drop datamap preagg16 on table PreAggMain")
- }
-
- test("test pre agg create table 11") {
- sql("create datamap preagg17 on table PreAggMain using 'preaggregate' as select a,min(b) from PreAggMain group by a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg17"), true, "preaggmain_a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg17"), true, "preaggmain_b_min")
- sql("drop datamap preagg17 on table PreAggMain")
- }
-
- test("test pre agg create table 12") {
- sql("create datamap preagg18 on table PreAggMain using 'preaggregate' as select a as a1,count(b) from PreAggMain group by a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg18"), true, "preaggmain_a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg18"), true, "preaggmain_b_count")
- sql("drop datamap preagg18 on table PreAggMain")
- }
-
- test("test pre agg create table 13") {
- val exception: Exception = intercept[MalformedCarbonCommandException] {
- sql(
- s"""
- | create datamap preagg19 on table PreAggMain
- | using 'preaggregate'
- | as select a as a1,count(distinct b)
- | from PreAggMain group by a
- """.stripMargin)
- }
- assert(exception.getMessage.equals("Distinct is not supported On Pre Aggregation"))
- }
-
- test("test pre agg create table 14") {
- val exception = intercept[MalformedCarbonCommandException] {
- sql(
- s"""
- | create datamap preagg20 on table PreAggMain
- | using 'preaggregate'
- | as select a as a1,sum(distinct b) from PreAggMain
- | group by a
- """.stripMargin)
- }
- assert(exception.getMessage.equals("Distinct is not supported On Pre Aggregation"))
- }
-
- test("test pre agg create table 15: don't support where") {
- intercept[Exception] {
- sql(
- s"""
- | create datamap preagg21 on table PreAggMain
- | using 'preaggregate'
- | as select a as a1,sum(b)
- | from PreAggMain
- | where a='vishal'
- | group by a
- """.stripMargin)
- }
- }
-
- test("test pre agg create table 16") {
- sql("create datamap agg0 on table mainTable using 'preaggregate' as select column4, sum(column4) from maintable group by column4")
- val df = sql("select * from maintable_agg0")
- val carbontable = getCarbonTable(df.queryExecution.analyzed)
- assert(carbontable.getAllMeasures.size()==2)
- assert(carbontable.getAllDimensions.size()==0)
- sql("drop datamap agg0 on table maintable")
- }
-
- test("test pre agg create table 17") {
- sql("create datamap agg0 on table mainTable using 'preaggregate' as select column1, sum(column1),column6, sum(column6) from maintable group by column6,column1")
- val df = sql("select * from maintable_agg0")
- val carbontable = getCarbonTable(df.queryExecution.analyzed)
- assert(carbontable.getAllMeasures.size()==2)
- assert(carbontable.getAllDimensions.size()==2)
- carbontable.getAllDimensions.asScala.foreach{ f =>
- assert(f.getEncoder.contains(Encoding.DICTIONARY))
- }
- sql("drop datamap agg0 on table maintable")
- }
-
- test("test pre agg create table 18") {
- sql("create datamap agg0 on table mainTable using 'preaggregate' as select column1, count(column1),column6, count(column6) from maintable group by column6,column1")
- val df = sql("select * from maintable_agg0")
- val carbontable = getCarbonTable(df.queryExecution.analyzed)
- assert(carbontable.getAllMeasures.size()==2)
- assert(carbontable.getAllDimensions.size()==2)
- carbontable.getAllDimensions.asScala.foreach{ f =>
- assert(f.getEncoder.contains(Encoding.DICTIONARY))
- }
- sql("drop datamap agg0 on table maintable")
- }
-
- test("test pre agg create table 19") {
- sql("create datamap agg0 on table mainTable using 'preaggregate' as select column3, sum(column3),column5, sum(column5) from maintable group by column3,column5")
- val df = sql("select * from maintable_agg0")
- val carbontable = getCarbonTable(df.queryExecution.analyzed)
- assert(carbontable.getAllMeasures.size()==2)
- assert(carbontable.getAllDimensions.size()==2)
- carbontable.getAllDimensions.asScala.foreach{ f =>
- assert(!f.getEncoder.contains(Encoding.DICTIONARY))
- }
- sql("drop datamap agg0 on table maintable")
- }
-
- test("test pre agg create table 20") {
- sql("create datamap agg0 on table mainTable using 'preaggregate' as select column3, sum(column3),column5, sum(column5) from maintable group by column3,column5,column2")
- val df = sql("select * from maintable_agg0")
- val carbontable = getCarbonTable(df.queryExecution.analyzed)
- assert(carbontable.getAllMeasures.size()==2)
- assert(carbontable.getAllDimensions.size()==3)
- carbontable.getAllDimensions.asScala.foreach{ f =>
- assert(!f.getEncoder.contains(Encoding.DICTIONARY))
- }
- sql("drop datamap agg0 on table maintable")
- }
-
- test("remove agg tables from show table command") {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,"false")
- sql("DROP TABLE IF EXISTS tbl_1")
- sql("DROP TABLE IF EXISTS sparktable")
- sql("create table if not exists tbl_1(imei string,age int,mac string ,prodate timestamp,update timestamp,gamepoint double,contrid double) stored by 'carbondata' ")
- sql("create table if not exists sparktable(a int,b string)")
- sql(
- s"""create datamap preagg_sum on table tbl_1 using 'preaggregate' as select mac,avg(age) from tbl_1 group by mac"""
- .stripMargin)
- sql(
- "create datamap agg2 on table tbl_1 using 'preaggregate' as select prodate, mac from tbl_1 group by prodate,mac")
- checkExistence(sql("show tables"), false, "tbl_1_preagg_sum","tbl_1_agg2_day","tbl_1_agg2_hour","tbl_1_agg2_month","tbl_1_agg2_year")
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,CarbonCommonConstants.CARBON_SHOW_DATAMAPS_DEFAULT)
- }
-
- test("test pre agg create table 22: create with preaggregate and granularity") {
- sql("DROP TABLE IF EXISTS maintabletime")
- sql(
- """
- | CREATE TABLE maintabletime(year INT,month INT,name STRING,salary INT,dob STRING)
- | STORED BY 'carbondata'
- | TBLPROPERTIES(
- | 'SORT_SCOPE'='Global_sort',
- | 'TABLE_BLOCKSIZE'='23',
- | 'SORT_COLUMNS'='month,year,name')
- """.stripMargin)
- sql("INSERT INTO maintabletime SELECT 10,11,'x',12,'2014-01-01 00:00:00'")
- sql(
- s"""
- | CREATE DATAMAP agg0 ON TABLE maintabletime
- | USING 'preaggregate'
- | AS SELECT dob,name FROM maintabletime
- | GROUP BY dob,name
- """.stripMargin)
- val e = intercept[MalformedCarbonCommandException] {
- sql(
- s"""
- | CREATE DATAMAP agg1 ON TABLE maintabletime
- | USING 'preaggregate'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dob',
- | 'SECOND_GRANULARITY'='1')
- | AS SELECT dob,name FROM maintabletime
- | GROUP BY dob,name
- """.stripMargin)
- }
- assert(e.getMessage.contains("Only 'path', 'partitioning' and 'long_string_columns' dmproperties "
- + "are allowed for this datamap"))
- sql("DROP TABLE IF EXISTS maintabletime")
- }
-
- test("test pre agg create table 22: using invalid datamap provider") {
- sql("DROP DATAMAP IF EXISTS agg0 ON TABLE maintable")
-
- val e = intercept[MalformedDataMapCommandException] {
- sql(
- """
- | CREATE DATAMAP agg0 ON TABLE mainTable
- | USING 'abc'
- | AS SELECT column3, SUM(column3),column5, SUM(column5)
- | FROM maintable
- | GROUP BY column3,column5,column2
- """.stripMargin)
- }
- assert(e.getMessage.contains(s"DataMap 'abc' not found"))
- sql("DROP DATAMAP IF EXISTS agg0 ON TABLE maintable")
- }
-
- test("test pre agg create table 24: remove agg tables from show table command") {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS, "false")
- sql("DROP TABLE IF EXISTS tbl_1")
- sql("create table if not exists tbl_1(imei string,age int,mac string ,prodate timestamp,update timestamp,gamepoint double,contrid double) stored by 'carbondata' ")
- sql("create datamap agg1 on table tbl_1 using 'preaggregate' as select mac, sum(age) from tbl_1 group by mac")
- sql("create table if not exists sparktable(imei string,age int,mac string ,prodate timestamp,update timestamp,gamepoint double,contrid double) ")
- checkExistence(sql("show tables"), false, "tbl_1_agg1")
- checkExistence(sql("show tables"), true, "sparktable", "tbl_1")
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS, CarbonCommonConstants.CARBON_SHOW_DATAMAPS_DEFAULT)
- }
-
- test("test pre agg create table 25: remove TimeSeries agg tables from show table command") {
- sql("DROP TABLE IF EXISTS tbl_1")
- sql("create table if not exists tbl_1(imei string,age int,mac string ,prodate timestamp,update timestamp,gamepoint double,contrid double) stored by 'carbondata' ")
- sql(
- "create datamap agg2 on table tbl_1 using 'preaggregate' as select prodate, mac from tbl_1 group by prodate,mac")
- checkExistence(sql("show tables"), false, "tbl_1_agg2_day","tbl_1_agg2_hour","tbl_1_agg2_month","tbl_1_agg2_year")
- }
-
- test("test pre agg create table 21: should support 'if not exists'") {
- sql(
- """
- | CREATE DATAMAP IF NOT EXISTS agg0 ON TABLE mainTable
- | USING 'preaggregate'
- | AS SELECT
- | column3,
- | sum(column3),
- | column5,
- | sum(column5)
- | FROM maintable
- | GROUP BY column3,column5,column2
- """.stripMargin)
- sql("DROP DATAMAP IF EXISTS agg0 ON TABLE maintable")
- }
-
- test("test pre agg create table 22: don't support create datamap if exists'") {
- val e: Exception = intercept[AnalysisException] {
- sql(
- """
- | CREATE DATAMAP IF EXISTS agg0 ON TABLE mainTable
- | USING 'preaggregate'
- | AS SELECT
- | column3,
- | sum(column3),
- | column5,
- | sum(column5)
- | FROM maintable
- | GROUP BY column3,column5,column2
- """.stripMargin)
- assert(true)
- }
- assert(e.getMessage.contains("identifier matching regex"))
- sql("DROP DATAMAP IF EXISTS agg0 ON TABLE maintable")
- }
-
- test("test show tables filtered with datamaps") {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,"false")
- sql("create datamap preagg1 on table PreAggMain using 'preaggregate' as select a,sum(b) from PreAggMain group by a")
- sql("show tables").show()
- checkExistence(sql("show tables"), false, "preaggmain_preagg1")
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,CarbonCommonConstants.CARBON_SHOW_DATAMAPS_DEFAULT)
- checkExistence(sql("show tables"), true, "preaggmain_preagg1")
- }
-
- test("test create main and preagg table of same name in two database") {
- sql("drop table if exists Preagg_twodb")
- sql("create table Preagg_twodb(name string, age int) stored by 'carbondata'")
- sql("create datamap sameName on table Preagg_twodb using 'preaggregate' as select sum(age) from Preagg_twodb")
- sql("create database otherDB")
- sql("use otherDB")
- sql("drop table if exists Preagg_twodb")
- sql("create table Preagg_twodb(name string, age int) stored by 'carbondata'")
- try {
- sql(
- "create datamap sameName on table Preagg_twodb using 'preaggregate' as select sum(age) from Preagg_twodb")
- assert(true)
- } catch {
- case ex: Exception =>
- assert(false)
- }
- sql("use default")
- }
-
- // TODO: to be confirmed
- test("test pre agg create table 26") {
- sql("drop datamap if exists preagg2 on table PreAggMain")
- sql("create datamap preagg2 on table PreAggMain using 'preaggregate' as select a as a1,sum(b) from PreAggMain group by a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg2"), true, "preaggmain_a")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg2"), true, "preaggmain_b_sum")
- checkExistence(sql("DESCRIBE FORMATTED PreAggMain_preagg2"), false, "preaggmain_a1")
- intercept[Exception] {
- sql("select a1 from PreAggMain_preagg2").show()
- }
- sql("drop datamap if exists preagg2 on table PreAggMain")
- }
-
- test("test pre agg create table 27: select * and no group by") {
- intercept[Exception] {
- sql(
- """
- | CREATE DATAMAP IF NOT EXISTS agg0 ON TABLE mainTable
- | USING 'preaggregate'
- | AS SELECT * FROM maintable
- """.stripMargin)
- }
- }
-
- // TODO : to be confirmed
- test("test pre agg create table 28: select *") {
- intercept[Exception] {
- sql(
- """
- | CREATE DATAMAP IF NOT EXISTS agg0 ON TABLE mainTable
- | USING 'preaggregate'
- | AS SELECT * FROM maintable
- | group by a
- """.stripMargin)
- }
- }
-
- test("test pre agg create table 29") {
- intercept[Exception] {
- sql(
- s"""
- | create datamap preagg21 on table PreAggMain2
- | using 'preaggregate'
- | as select a as a1,sum(b)
- | from PreAggMain2
- | where a>'vishal'
- | group by a
- """.stripMargin)
- }
- }
-
- test("test pre agg create table 30: DESCRIBE FORMATTED") {
- dropDataMaps("PreAggMain", "preagg2")
- intercept[Exception] {
- sql("DESCRIBE FORMATTED PreAggMain_preagg2").show()
- }
- }
-
- test("test codegen issue with preaggregate") {
- sql("DROP TABLE IF EXISTS PreAggMain")
- sql("CREATE TABLE PreAggMain (id Int, date date, country string, phonetype string, " +
- "serialname String,salary int ) STORED BY 'org.apache.carbondata.format' " +
- "tblproperties('dictionary_include'='country')")
- sql("create datamap PreAggSum on table PreAggMain using 'preaggregate' as " +
- "select country,sum(salary) as sum from PreAggMain group by country")
- sql("create datamap PreAggAvg on table PreAggMain using 'preaggregate' as " +
- "select country,avg(salary) as avg from PreAggMain group by country")
- sql("create datamap PreAggCount on table PreAggMain using 'preaggregate' as " +
- "select country,count(salary) as count from PreAggMain group by country")
- sql("create datamap PreAggMin on table PreAggMain using 'preaggregate' as " +
- "select country,min(salary) as min from PreAggMain group by country")
- sql("create datamap PreAggMax on table PreAggMain using 'preaggregate' as " +
- "select country,max(salary) as max from PreAggMain group by country")
- sql(s"LOAD DATA INPATH '$integrationPath/spark-common-test/src/test/resources/source.csv' " +
- s"into table PreAggMain")
- checkExistence(sql("select t1.country,sum(id) from PreAggMain t1 join (select " +
- "country as newcountry,sum(salary) as sum from PreAggMain group by country)" +
- "t2 on t1.country=t2.newcountry group by country"), true, "france")
- sql("DROP TABLE IF EXISTS PreAggMain")
- }
-
- // TODO: Need to Fix
- ignore("test creation of multiple preaggregate of same name concurrently") {
- sql("DROP TABLE IF EXISTS tbl_concurr")
- sql(
- "create table if not exists tbl_concurr(imei string,age int,mac string ,prodate timestamp," +
- "update timestamp,gamepoint double,contrid double) stored by 'carbondata' ")
-
- var executorService: ExecutorService = Executors.newCachedThreadPool()
- val tasks = new util.ArrayList[Callable[String]]()
- var i = 0
- val count = 5
- while (i < count) {
- tasks
- .add(new QueryTask(
- s"""create datamap agg_concu1 on table tbl_concurr using
- |'preaggregate' as select prodate, mac from tbl_concurr group by prodate,mac"""
- .stripMargin))
- i = i + 1
- }
- executorService.invokeAll(tasks).asScala
- executorService.awaitTermination(5, TimeUnit.MINUTES)
- checkExistence(sql("show tables"), true, "agg_concu1", "tbl_concurr")
- executorService.shutdown()
- }
-
- class QueryTask(query: String) extends Callable[String] {
- override def call(): String = {
- var result = "SUCCESS"
- try {
- sql(query).collect()
- } catch {
- case exception: Exception => LOGGER.error(exception.getMessage)
- }
- result
- }
- }
-
-
- def getCarbonTable(plan: LogicalPlan) : CarbonTable ={
- var carbonTable : CarbonTable = null
- plan.transform {
- // first check if any preaTable1 scala function is applied it is present is in plan
- // then call is from create preaTable1regate table class so no need to transform the query plan
- case ca:CarbonRelation =>
- if (ca.isInstanceOf[CarbonDatasourceHadoopRelation]) {
- val relation = ca.asInstanceOf[CarbonDatasourceHadoopRelation]
- carbonTable = relation.carbonTable
- }
- ca
- case logicalRelation:LogicalRelation =>
- if(logicalRelation.relation.isInstanceOf[CarbonDatasourceHadoopRelation]) {
- val relation = logicalRelation.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
- carbonTable = relation.carbonTable
- }
- logicalRelation
- }
- carbonTable
- }
-
- override def afterAll {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,
- CarbonCommonConstants.CARBON_SHOW_DATAMAPS_DEFAULT)
- sql("drop database if exists otherDB cascade")
- sql("drop table if exists maintable")
- sql("drop table if exists PreAggMain")
- sql("drop table if exists PreAggMain1")
- sql("drop table if exists maintabletime")
- sql("drop table if exists showTables")
- sql("drop table if exists Preagg_twodb")
- sql("DROP TABLE IF EXISTS tbl_1")
- sql("DROP TABLE IF EXISTS sparktable")
- }
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggStreaming.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggStreaming.scala
deleted file mode 100644
index 28b7546..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggStreaming.scala
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.integration.spark.testsuite.preaggregate
-
-import org.apache.spark.sql.CarbonDatasourceHadoopRelation
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Union}
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.{BeforeAndAfterAll, Ignore}
-
-
-class TestPreAggStreaming extends QueryTest with BeforeAndAfterAll {
-
-
- override def beforeAll: Unit = {
- dropAll
- sql("CREATE TABLE mainTable(id int, name string, city string, age string) STORED BY 'org.apache.carbondata.format' tblproperties('streaming'='true')")
- sql("create datamap agg0 on table mainTable using 'preaggregate' as select name from mainTable group by name")
- sql("create datamap agg1 on table mainTable using 'preaggregate' as select name,sum(age) from mainTable group by name")
- sql("create datamap agg2 on table mainTable using 'preaggregate' as select name,avg(age) from mainTable group by name")
- sql("create datamap agg3 on table mainTable using 'preaggregate' as select name,sum(CASE WHEN age=35 THEN id ELSE 0 END) from mainTable group by name")
- sql("CREATE TABLE mainTableStreamingOne(id int, name string, city string, age smallint) STORED BY 'org.apache.carbondata.format' tblproperties('streaming'='true')")
- sql("create datamap aggStreamingAvg on table mainTableStreamingOne using 'preaggregate' as select name,avg(age) from mainTableStreamingOne group by name")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table mainTable")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table mainTableStreamingOne")
- sql("CREATE TABLE origin(id int, name string, city string, age string) STORED BY 'org.apache.carbondata.format' tblproperties('streaming'='true')")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table origin")
- sql("CREATE TABLE binary_stream(id int, label boolean, name string,image binary,autoLabel boolean) STORED BY 'org.apache.carbondata.format' tblproperties('streaming'='true')")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/binaryDataBase64.csv' into table binary_stream OPTIONS('header'='false','DELIMITER'=',','binary_decoder'='baSe64')")
- sql("create datamap agg0 on table binary_stream using 'preaggregate' as select name from binary_stream group by name")
- }
-
- test("Test Pre Agg Streaming with project column and group by") {
- val df = sql("select name from maintable group by name")
- assert(validateStreamingTablePlan(df.queryExecution.analyzed))
- checkAnswer(df, sql("select name from origin group by name"))
- }
-
- test("Test binary with stream and preaggregate") {
- val df = sql("select name from binary_stream group by name")
- df.collect()
- assert(validateStreamingTablePlan(df.queryExecution.analyzed))
- checkAnswer(df, sql("select name from binary_stream group by name"))
- }
-
- test("Test Pre Agg Streaming table Agg Sum Aggregation") {
- val df = sql("select name, sum(age) from maintable group by name")
- assert(validateStreamingTablePlan(df.queryExecution.analyzed))
- checkAnswer(df, sql("select name, sum(age) from origin group by name"))
- }
-
- test("Test Pre Agg Streaming table with UDF") {
- val df = sql("select substring(name,1,1), sum(age) from maintable group by substring(name,1,1)")
- assert(validateStreamingTablePlan(df.queryExecution.analyzed))
- checkAnswer(df, sql("select substring(name,1,1), sum(age) from origin group by substring(name,1,1)"))
- }
-
- test("Test Pre Agg Streaming table with UDF Only in group by") {
- val df = sql("select sum(age) from maintable group by substring(name,1,1)")
- assert(validateStreamingTablePlan(df.queryExecution.analyzed))
- checkAnswer(df, sql("select sum(age) from origin group by substring(name,1,1)"))
- }
-
- test("Test Pre Agg Streaming table With Sum Aggregation And Order by") {
- val df = sql("select name, sum(age) from maintable group by name order by name")
- assert(validateStreamingTablePlan(df.queryExecution.analyzed))
- checkAnswer(df, sql("select name, sum(age) from origin group by name order by name"))
- }
-
- test("Test Pre Agg Streaming table With Avg Aggregation") {
- val df = sql("select name, avg(age) from maintable group by name order by name")
- assert(validateStreamingTablePlan(df.queryExecution.analyzed))
- checkAnswer(df, sql("select name, avg(age) from origin group by name order by name"))
- }
-
- test("Test Pre Agg Streaming table With Expression Aggregation") {
- val df = sql("select name, sum(CASE WHEN age=35 THEN id ELSE 0 END) from maintable group by name order by name")
- assert(validateStreamingTablePlan(df.queryExecution.analyzed))
- checkAnswer(df, sql("select name, sum(CASE WHEN age=35 THEN id ELSE 0 END) from origin group by name order by name"))
- }
-
- test("Test Pre Agg Streaming table With only aggregate expression and group by") {
- val df = sql("select sum(age) from maintable group by name")
- assert(validateStreamingTablePlan(df.queryExecution.analyzed))
- checkAnswer(df, sql("select sum(age) from origin group by name"))
- }
-
- test("Test Pre Agg Streaming table With small int and avg") {
- val df = sql("select name, avg(age) from mainTableStreamingOne group by name")
- assert(validateStreamingTablePlan(df.queryExecution.analyzed))
- checkAnswer(df, sql("select name, avg(age) from origin group by name"))
- }
-
- /**
- * Below method will be used validate whether plan is already updated in case of streaming table
- * In case of streaming table it will add UnionNode to get the data from fact and aggregate both
- * as aggregate table will be updated after each handoff.
- * So if plan is already updated no need to transform the plan again
- * @param logicalPlan
- * query plan
- * @return whether need to update the query plan or not
- */
- def validateStreamingTablePlan(logicalPlan: LogicalPlan) : Boolean = {
- var isChildTableExists: Boolean = false
- logicalPlan.transform {
- case union @ Union(Seq(plan1, plan2)) =>
- plan2.collect{
- case logicalRelation: LogicalRelation if
- logicalRelation.relation.isInstanceOf[CarbonDatasourceHadoopRelation] &&
- logicalRelation.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonTable
- .isChildDataMap =>
- isChildTableExists = true
- }
- union
- }
- isChildTableExists
- }
-
- private def dropAll: Unit = {
- sql("drop table if exists mainTable")
- sql("drop table if exists mainTableStreamingOne")
- sql("drop table if exists origin")
- sql("drop table if exists binary_stream")
- }
-
- override def afterAll: Unit = {
- dropAll
- }
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateCompaction.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateCompaction.scala
deleted file mode 100644
index 7bf0c35..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateCompaction.scala
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.integration.spark.testsuite.preaggregate
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
-import org.scalatest.Matchers._
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-class TestPreAggregateCompaction extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
-
- val testData = s"$resourcesPath/sample.csv"
-
- override def beforeEach(): Unit = {
- sql("drop database if exists compaction cascade")
- sql("create database if not exists compaction")
- sql("use compaction")
- sql("create table testtable (id int, name string, city string, age int) STORED BY 'org.apache.carbondata.format'")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(
- s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id,sum(age) from maintable group by id"""
- .stripMargin)
- sql(
- s"""create datamap preagg_avg on table maintable using 'preaggregate' as select id,avg(age) from maintable group by id"""
- .stripMargin)
- }
-
- test("test if pre-agg table is compacted with parent table minor compaction") {
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql("insert into testtable select * from maintable")
- val sumResult = sql("select id, sum(age) from testtable group by id").collect()
- val avgResult = sql("select id, sum(age), count(age) from testtable group by id").collect()
- sql("alter table maintable compact 'minor'")
- val segmentNamesSum = sql("show segments for table maintable_preagg_sum").collect().map(_.get(0).toString)
- segmentNamesSum should equal (Array("3", "2", "1", "0.1", "0"))
- checkAnswer(sql("select * from maintable_preagg_sum"), sumResult)
- val segmentNamesAvg = sql("show segments for table maintable_preagg_avg").collect().map(_.get(0).toString)
- segmentNamesAvg should equal (Array("3", "2", "1", "0.1", "0"))
- checkAnswer(sql("select * from maintable_preagg_avg"), avgResult)
- }
-
- test("test if pre-agg table is compacted with parent table major compaction") {
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql("alter table maintable compact 'major'")
- sql("insert into testtable select * from maintable")
- val sumResult = sql("select id, sum(age) from testtable group by id").collect()
- val avgResult = sql("select id, sum(age), count(age) from testtable group by id").collect()
- sql("alter table maintable compact 'minor'")
- val segmentNamesSum = sql("show segments for table maintable_preagg_sum").collect().map(_.get(0).toString)
- segmentNamesSum should equal (Array("3", "2", "1", "0.1", "0"))
- checkAnswer(sql("select * from maintable_preagg_sum"), sumResult)
- val segmentNamesAvg = sql("show segments for table maintable_preagg_avg").collect().map(_.get(0).toString)
- segmentNamesAvg should equal (Array("3", "2", "1", "0.1", "0"))
- checkAnswer(sql("select * from maintable_preagg_avg"), avgResult)
- }
-
- test("test if 2nd level minor compaction is successful for pre-agg table") {
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql("alter table maintable compact 'minor'")
- var segmentNamesSum = sql("show segments for table maintable_preagg_sum").collect().map(_.get(0).toString)
- segmentNamesSum should equal (Array("3", "2", "1", "0.1", "0"))
- sql("insert into testtable select * from maintable")
- var sumResult = sql("select id, sum(age) from testtable group by id").collect()
- var avgResult = sql("select id, sum(age), count(age) from testtable group by id").collect()
- checkAnswer(sql("select * from maintable_preagg_sum"), sumResult)
- var segmentNamesAvg = sql("show segments for table maintable_preagg_avg").collect().map(_.get(0).toString)
- segmentNamesAvg should equal (Array("3", "2", "1", "0.1", "0"))
- checkAnswer(sql("select * from maintable_preagg_avg"), avgResult)
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql("alter table maintable compact 'minor'")
- sql("insert overwrite table testtable select * from maintable")
- sumResult = sql("select id, sum(age) from testtable group by id").collect()
- avgResult = sql("select id, sum(age), count(age) from testtable group by id").collect()
- segmentNamesSum = sql("show segments for table maintable_preagg_sum").collect().map(_.get(0).toString)
- segmentNamesSum.sorted should equal (Array("0", "0.1", "1", "2", "3", "4", "4.1", "5", "6", "7"))
- checkAnswer(sql("select maintable_id, sum(maintable_age_sum) from maintable_preagg_sum group by maintable_id"), sumResult)
- segmentNamesAvg = sql("show segments for table maintable_preagg_avg").collect().map(_.get(0).toString)
- segmentNamesAvg.sorted should equal (Array("0", "0.1", "1", "2", "3", "4", "4.1", "5", "6", "7"))
- checkAnswer(sql("select maintable_id, sum(maintable_age_sum), sum(maintable_age_count) from maintable_preagg_avg group by maintable_id"), avgResult)
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql("alter table maintable compact 'minor'")
- sql("insert overwrite table testtable select * from maintable")
- sumResult = sql("select id, sum(age) from testtable group by id").collect()
- avgResult = sql("select id, sum(age), count(age) from testtable group by id").collect()
- segmentNamesSum = sql("show segments for table maintable_preagg_sum").collect().map(_.get(0).toString)
- segmentNamesSum should equal (Array("11", "10", "9", "8.1", "8", "7", "6", "5", "4.1", "4", "3", "2", "1", "0.2", "0.1", "0"))
- checkAnswer(sql("select maintable_id, sum(maintable_age_sum) from maintable_preagg_sum group by maintable_id"), sumResult)
- segmentNamesAvg = sql("show segments for table maintable_preagg_avg").collect().map(_.get(0).toString)
- segmentNamesAvg should equal (Array("11", "10", "9", "8.1", "8", "7", "6", "5", "4.1", "4", "3", "2", "1", "0.2", "0.1", "0"))
- checkAnswer(sql("select maintable_id, sum(maintable_age_sum), sum(maintable_age_count) from maintable_preagg_avg group by maintable_id"), avgResult)
- }
-
- test("test direct minor compaction on pre-agg tables") {
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql("alter table maintable_preagg_sum compact 'minor'")
- var segmentNamesSum = sql("show segments for table maintable_preagg_sum").collect().map(_.get(0).toString)
- segmentNamesSum should equal (Array("3", "2", "1", "0.1", "0"))
- sql("insert into testtable select * from maintable")
- var sumResult = sql("select id, sum(age) from testtable group by id").collect()
- checkAnswer(sql("select * from maintable_preagg_sum"), sumResult)
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql("alter table maintable_preagg_sum compact 'minor'")
- sql("insert overwrite table testtable select * from maintable")
- sumResult = sql("select id, sum(age) from testtable group by id").collect()
- segmentNamesSum = sql("show segments for table maintable_preagg_sum").collect().map(_.get(0).toString)
- segmentNamesSum.sorted should equal (Array("0", "0.1", "1", "2", "3", "4", "4.1", "5", "6", "7"))
- checkAnswer(sql("select maintable_id, sum(maintable_age_sum) from maintable_preagg_sum group by maintable_id"), sumResult)
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql("alter table maintable_preagg_sum compact 'minor'")
- sql("insert overwrite table testtable select * from maintable")
- sumResult = sql("select id, sum(age) from testtable group by id").collect()
- segmentNamesSum = sql("show segments for table maintable_preagg_sum").collect().map(_.get(0).toString)
- segmentNamesSum should equal (Array("11", "10", "9", "8.1", "8", "7", "6", "5", "4.1", "4", "3", "2", "1", "0.2", "0.1", "0"))
- checkAnswer(sql("select maintable_id, sum(maintable_age_sum) from maintable_preagg_sum group by maintable_id"), sumResult)
- val mainTableSegment = sql("SHOW SEGMENTS FOR TABLE maintable")
- val SegmentSequenceIds = mainTableSegment.collect().map { each => (each.toSeq) (0) }
- assert(!SegmentSequenceIds.contains("0.1"))
- }
-
- test("test if minor/major compaction is successful for pre-agg table") {
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql("alter table maintable_preagg_sum compact 'minor'")
- var segmentNamesSum = sql("show segments for table maintable_preagg_sum").collect().map(_.get(0).toString)
- segmentNamesSum should equal (Array("3","2","1","0.1", "0"))
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql("alter table maintable_preagg_sum compact 'major'")
- segmentNamesSum = sql("show segments for table maintable_preagg_sum").collect().map(_.get(0).toString)
- segmentNamesSum.sorted should equal (Array("0", "0.1", "0.2", "1", "2", "3", "4", "5", "6", "7"))
- }
-
- test("test auto compaction on aggregate table") {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- val segmentNamesSum = sql("show segments for table maintable_preagg_sum").collect().map(_.get(0).toString)
- segmentNamesSum.sorted should equal (Array("0", "0.1", "1", "2", "3"))
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
- }
-
- test("test minor compaction on Pre-agg tables after multiple loads") {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql("alter table maintable compact 'minor'")
- assert(sql("show segments for table maintable").collect().map(_.get(1).toString.toLowerCase).contains("compacted"))
- }
-
- override def afterAll(): Unit = {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
- sql("drop database if exists compaction cascade")
- sql("use default")
- }
-
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
deleted file mode 100644
index f99841c..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.integration.spark.testsuite.preaggregate
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.common.exceptions.sql.NoSuchDataMapException
-import org.apache.carbondata.spark.exception.ProcessMetaDataException
-
-class TestPreAggregateDrop extends QueryTest with BeforeAndAfterAll {
-
- override def beforeAll {
- sql("drop table if exists maintable")
- sql("create table maintable (a string, b string, c string) stored by 'carbondata'")
- }
-
- test("create and drop preaggregate table") {
- sql(
- "create datamap preagg1 on table maintable using 'preaggregate' as select" +
- " a,sum(b) from maintable group by a")
- sql("drop datamap if exists preagg1 on table maintable")
- checkExistence(sql("SHOW DATAMAP ON TABLE maintable"), false, "maintable_preagg1")
- }
-
- test("dropping 1 aggregate table should not drop others") {
- sql(
- "create datamap preagg1 on table maintable using 'preaggregate' as select" +
- " a,sum(b) from maintable group by a")
- sql(
- "create datamap preagg2 on table maintable using 'preaggregate' as select" +
- " a,sum(c) from maintable group by a")
- sql("drop datamap if exists preagg2 on table maintable")
- val showdatamaps =sql("show datamap on table maintable")
- checkExistence(showdatamaps, false, "maintable_preagg2")
- checkExistence(showdatamaps, true, "maintable_preagg1")
- }
-
- test("drop datamap which is not existed") {
- intercept[NoSuchDataMapException] {
- sql("drop datamap newpreagg on table maintable")
- }
- }
-
- test("drop datamap with same name on different tables") {
- sql("drop table if exists maintable1")
- sql("create datamap preagg_same on table maintable using 'preaggregate' as select" +
- " a,sum(c) from maintable group by a")
- sql("create table maintable1 (a string, b string, c string) stored by 'carbondata'")
- sql("create datamap preagg_same on table maintable1 using 'preaggregate' as select" +
- " a,sum(c) from maintable1 group by a")
-
- sql("drop datamap preagg_same on table maintable")
- val showDataMaps = sql("SHOW DATAMAP ON TABLE maintable1")
- checkExistence(showDataMaps, false, "maintable_preagg_same")
- checkExistence(showDataMaps, true, "maintable1_preagg_same")
- sql("drop datamap preagg_same on table maintable1")
- checkExistence(sql("SHOW DATAMAP ON TABLE maintable1"), false, "maintable1_preagg_same")
- sql("drop table if exists maintable1")
- }
-
- test("drop datamap and create again with same name") {
- sql("create datamap preagg_same1 on table maintable using 'preaggregate' as select" +
- " a,sum(c) from maintable group by a")
-
- sql("drop datamap preagg_same1 on table maintable")
- checkExistence(sql("SHOW DATAMAP ON TABLE maintable"), false, "maintable_preagg_same1")
- sql("create datamap preagg_same1 on table maintable using 'preaggregate' as select" +
- " a,sum(c) from maintable group by a")
- val showDatamaps =sql("show datamap on table maintable")
- checkExistence(showDatamaps, true, "maintable_preagg_same1")
- sql("drop datamap preagg_same1 on table maintable")
- }
-
- test("drop main table and check if preaggreagte is deleted") {
- sql(
- "create datamap preagg2 on table maintable using 'preaggregate' as select" +
- " a,sum(c) from maintable group by a")
- sql("drop table if exists maintable")
- checkExistence(sql("show tables"), false, "maintable_preagg1", "maintable", "maintable_preagg2")
- }
-
- test("drop datamap with 'if exists' when datamap not exists") {
- sql("DROP TABLE IF EXISTS maintable")
- sql("CREATE TABLE maintable (a STRING, b STRING, c STRING) STORED BY 'carbondata'")
- sql("DROP DATAMAP IF EXISTS not_exists_datamap ON TABLE maintable")
- checkExistence(sql("DESCRIBE FORMATTED maintable"), false, "not_exists_datamap")
- }
-
- test("drop datamap without 'if exists' when datamap not exists") {
- sql("DROP TABLE IF EXISTS maintable")
- sql("CREATE TABLE maintable (a STRING, b STRING, c STRING) STORED BY 'carbondata'")
- sql("DROP DATAMAP IF EXISTS not_exists_datamap ON TABLE maintable")
- val e = intercept[NoSuchDataMapException] {
- sql("DROP DATAMAP not_exists_datamap ON TABLE maintable")
- }
- assert(e.getMessage.equals(
- "Datamap with name not_exists_datamap does not exist"))
- }
-
- test("drop datamap without 'if exists' when main table not exists") {
- sql("DROP TABLE IF EXISTS maintable")
- val e = intercept[ProcessMetaDataException] {
- sql("DROP DATAMAP preagg3 ON TABLE maintable")
- }
- assert(e.getMessage.contains("Table or view 'maintable' not found in"))
- }
-
- test("drop datamap with 'if exists' when main table not exists") {
- sql("DROP TABLE IF EXISTS maintable")
- val e = intercept[ProcessMetaDataException] {
- sql("DROP DATAMAP IF EXISTS preagg3 ON TABLE maintable")
- }
- assert(e.getMessage.contains("Table or view 'maintable' not found in"))
- }
-
- test("drop preaggregate datamap whose main table has other type datamaps") {
- sql("drop table if exists maintable")
- sql("create table maintable (a string, b string, c string) stored by 'carbondata'")
- sql(
- "create datamap bloom1 on table maintable using 'bloomfilter'" +
- " dmproperties('index_columns'='a')")
- sql(
- "create datamap preagg1 on table maintable using 'preaggregate' as select" +
- " a,sum(b) from maintable group by a")
- sql("drop datamap if exists bloom1 on table maintable")
- sql("drop datamap if exists preagg1 on table maintable")
- checkExistence(sql("SHOW DATAMAP ON TABLE maintable"), false, "preagg1", "bloom1")
- sql(
- "create datamap bloom1 on table maintable using 'bloomfilter'" +
- " dmproperties('index_columns'='a')")
- sql(
- "create datamap preagg1 on table maintable using 'preaggregate' as select" +
- " a,sum(b) from maintable group by a")
- sql("drop datamap if exists preagg1 on table maintable")
- sql("drop datamap if exists bloom1 on table maintable")
- checkExistence(sql("SHOW DATAMAP ON TABLE maintable"), false, "preagg1", "bloom1")
- sql(
- "create datamap bloom1 on table maintable using 'bloomfilter'" +
- " dmproperties('index_columns'='a')")
- sql(
- "create datamap preagg1 on table maintable using 'preaggregate' as select" +
- " a,sum(b) from maintable group by a")
- sql("drop table if exists maintable")
- checkExistence(sql("show tables"), false, "maintable_preagg1", "maintable")
- checkExistence(sql("show datamap"), false, "preagg1", "bloom1")
- }
-
- override def afterAll() {
- sql("drop table if exists maintable")
- sql("drop table if exists maintable1")
- }
-
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateExpressions.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateExpressions.scala
deleted file mode 100644
index a7511fd..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateExpressions.scala
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.integration.spark.testsuite.preaggregate
-
-import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.sql.hive.CarbonRelation
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-class TestPreAggregateExpressions extends QueryTest with BeforeAndAfterAll {
-
- override def beforeAll: Unit = {
- sql("DROP TABLE IF EXISTS mainTable")
- sql("CREATE TABLE mainTable(id int, name string, city string, age string) STORED BY 'org.apache.carbondata.format'")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table mainTable")
- }
-
- test("test pre agg create table with expression 1") {
- sql(
- s"""
- | CREATE DATAMAP agg0 ON TABLE mainTable USING 'preaggregate' AS
- | SELECT name,
- | count(age)
- | FROM mainTable GROUP BY name
- | """.stripMargin)
- checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0"), true, "maintable_age_count")
- }
-
- test("test pre agg create table with expression 2") {
- sql(
- s"""
- | CREATE DATAMAP agg1 ON TABLE mainTable USING 'preaggregate' AS
- | SELECT name,
- | sum(CASE WHEN age=35 THEN id ELSE 0 END)
- | FROM mainTable GROUP BY name
- | """.stripMargin)
- checkExistence(sql("DESCRIBE FORMATTED mainTable_agg1"), true, "maintable_column_0_sum")
- }
-
- test("test pre agg create table with expression 3") {
- sql(
- s"""
- | CREATE DATAMAP agg2 ON TABLE mainTable USING 'preaggregate' AS
- | SELECT name,
- | sum(CASE WHEN age=35 THEN id ELSE 0 END),
- | city
- | FROM mainTable GROUP BY name,city
- | """.stripMargin)
- checkExistence(sql("DESCRIBE FORMATTED mainTable_agg2"), true, "maintable_column_0_sum")
- }
-
- test("test pre agg create table with expression 4") {
- sql(
- s"""
- | CREATE DATAMAP agg3 ON TABLE mainTable USING 'preaggregate' AS
- | SELECT name,
- | sum(CASE WHEN age=27 THEN id ELSE 0 END)
- | FROM mainTable GROUP BY name
- | """.stripMargin)
- checkExistence(sql("DESCRIBE FORMATTED mainTable_agg3"), true, "maintable_column_0_sum")
- }
-
- test("test pre agg create table with expression 5") {
- sql(
- s"""
- | CREATE DATAMAP agg4 ON TABLE mainTable USING 'preaggregate' AS
- | SELECT name,
- | sum(CASE WHEN age=27 THEN id ELSE 0 END),
- | SUM(CASE WHEN age=35 THEN id ELSE 0 END)
- | FROM mainTable GROUP BY name
- | """.stripMargin)
- checkExistence(sql("DESCRIBE FORMATTED mainTable_agg4"), true, "maintable_column_0_sum")
- checkExistence(sql("DESCRIBE FORMATTED mainTable_agg4"), true, "maintable_column_1_sum")
- }
-
- test("test pre agg create table with expression 6") {
- sql(
- s"""
- | CREATE DATAMAP agg5 ON TABLE mainTable USING 'preaggregate' AS
- | SELECT name,
- | COUNT(CASE WHEN age=27 THEN(CASE WHEN name='eason' THEN id ELSE 0 END) ELSE 0 END)
- | FROM mainTable GROUP BY name
- | """.stripMargin)
- checkExistence(sql("DESCRIBE FORMATTED mainTable_agg5"), true, "maintable_column_0_count")
- }
-
- test("test pre agg table selection with expression 1") {
- val df = sql("select name as NewName, count(age) as sum from mainTable group by name order by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0")
- }
-
- test("test pre agg table selection with expression 2") {
- val df = sql("select name as NewName, sum(case when age=35 then id else 0 end) as sum from mainTable group by name order by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg1")
- }
-
- test("test pre agg table selection with expression 3") {
- val df = sql("select sum(case when age=35 then id else 0 end) from maintable")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg1")
- checkAnswer(df, Seq(Row(6.0)))
- }
-
- test("test pre agg table selection with expression 4") {
- val df = sql("select sum(case when age=27 then id else 0 end) from maintable")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg3")
- checkAnswer(df, Seq(Row(2.0)))
- }
-
- test("test pre agg table selection with expression 5") {
- val df = sql("select sum(case when age=27 then id else 0 end), sum(case when age=35 then id else 0 end) from maintable")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg4")
- checkAnswer(df, Seq(Row(2.0,6.0)))
- }
-
- /**
- * Below method will be used to validate the table name is present in the plan or not
- *
- * @param plan query plan
- * @param actualTableName table name to be validated
- */
- def preAggTableValidator(plan: LogicalPlan, actualTableName: String) : Unit ={
- var isValidPlan = false
- plan.transform {
- // first check if any preaTable1 scala function is applied it is present is in plan
- // then call is from create preaTable1regate table class so no need to transform the query plan
- case ca:CarbonRelation =>
- if (ca.isInstanceOf[CarbonDatasourceHadoopRelation]) {
- val relation = ca.asInstanceOf[CarbonDatasourceHadoopRelation]
- if(relation.carbonTable.getTableName.equalsIgnoreCase(actualTableName)) {
- isValidPlan = true
- }
- }
- ca
- case logicalRelation:LogicalRelation =>
- if(logicalRelation.relation.isInstanceOf[CarbonDatasourceHadoopRelation]) {
- val relation = logicalRelation.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
- if(relation.carbonTable.getTableName.equalsIgnoreCase(actualTableName)) {
- isValidPlan = true
- }
- }
- logicalRelation
- }
- if(!isValidPlan) {
- assert(false)
- } else {
- assert(true)
- }
- }
-
- test("Test Pre_aggregate with decimal column with order by") {
- sql("drop table if exists maintable")
- sql("create table maintable(name string, decimal_col decimal(30,16)) stored by 'carbondata'")
- sql("insert into table maintable select 'abc',452.564")
- sql(
- "create datamap ag1 on table maintable using 'preaggregate' as select name,avg(decimal_col)" +
- " from maintable group by name")
- checkAnswer(sql("select avg(decimal_col) from maintable group by name order by name"),
- Seq(Row(452.56400000000000000000)))
- }
-
- override def afterAll: Unit = {
- sql("DROP TABLE IF EXISTS mainTable")
- }
-
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
deleted file mode 100644
index 75d71ec..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
+++ /dev/null
@@ -1,1039 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.integration.spark.testsuite.preaggregate
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.test.Spark2TestQueryExecutor
-import org.apache.spark.util.SparkUtil4Test
-import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
-
-import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.util.SparkQueryTest
-
-class TestPreAggregateLoad extends SparkQueryTest with BeforeAndAfterAll with BeforeAndAfterEach{
-
- val testData = s"$resourcesPath/sample.csv"
- val p1 = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD,
- CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD)
-
- override def beforeAll(): Unit = {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
- .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD,
- CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD)
-
- SparkUtil4Test.createTaskMockUp(sqlContext)
- sql("DROP TABLE IF EXISTS maintable")
- }
-
- override protected def afterAll(): Unit = {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
- CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE)
- .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD, p1)
- sql("DROP TABLE IF EXISTS y ")
- sql("DROP TABLE IF EXISTS maintable")
- sql("DROP TABLE IF EXISTS maintbl")
- sql("DROP TABLE IF EXISTS main_table")
- }
-
- override protected def beforeEach(): Unit = {
- sql("DROP TABLE IF EXISTS main_table")
- sql("DROP TABLE IF EXISTS segmaintable")
- }
-
- private def createAllAggregateTables(parentTableName: String, columnName: String = "age"): Unit = {
- sql(
- s"""
- | create datamap preagg_sum
- | on table $parentTableName
- | using 'preaggregate'
- | as select id,sum($columnName)
- | from $parentTableName
- | group by id
- """.stripMargin)
- sql(
- s"""create datamap preagg_avg on table $parentTableName using 'preaggregate' as select id,avg($columnName) from $parentTableName group by id"""
- .stripMargin)
- sql(
- s"""create datamap preagg_count on table $parentTableName using 'preaggregate' as select id,count($columnName) from $parentTableName group by id"""
- .stripMargin)
- sql(
- s"""create datamap preagg_min on table $parentTableName using 'preaggregate' as select id,min($columnName) from $parentTableName group by id"""
- .stripMargin)
- sql(
- s"""create datamap preagg_max on table $parentTableName using 'preaggregate' as select id,max($columnName) from $parentTableName group by id"""
- .stripMargin)
- }
-
- test("test load into main table with pre-aggregate table") {
- sql("DROP TABLE IF EXISTS maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- createAllAggregateTables("maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- checkAnswer(sql(s"select * from maintable_preagg_sum"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
- checkAnswer(sql(s"select * from maintable_preagg_avg"),
- Seq(Row(1, 31, 1), Row(2, 27, 1), Row(3, 70, 2), Row(4, 55, 2)))
- checkAnswer(sql(s"select * from maintable_preagg_count"),
- Seq(Row(1, 1), Row(2, 1), Row(3, 2), Row(4, 2)))
- checkAnswer(sql(s"select * from maintable_preagg_min"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 35), Row(4, 26)))
- checkAnswer(sql(s"select * from maintable_preagg_max"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 35), Row(4, 29)))
- sql("drop table if exists maintable")
- }
-
- test("test load into main table with pre-aggregate table with dictionary_include") {
- sql("drop table if exists maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('dictionary_include'='id')
- """.stripMargin)
- createAllAggregateTables("maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- checkAnswer(sql(s"select * from maintable_preagg_sum"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
- checkAnswer(sql(s"select * from maintable_preagg_avg"),
- Seq(Row(1, 31, 1), Row(2, 27, 1), Row(3, 70, 2), Row(4, 55,2)))
- checkAnswer(sql(s"select * from maintable_preagg_count"),
- Seq(Row(1, 1), Row(2, 1), Row(3, 2), Row(4, 2)))
- checkAnswer(sql(s"select * from maintable_preagg_min"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 35), Row(4, 26)))
- checkAnswer(sql(s"select * from maintable_preagg_max"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 35), Row(4, 29)))
- sql("drop table if exists maintable")
- }
-
- test("test load into main table with pre-aggregate table with single_pass") {
- sql("drop table if exists maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('dictionary_include'='id')
- """.stripMargin)
- createAllAggregateTables("maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable options('single_pass'='true')")
- checkAnswer(sql(s"select * from maintable_preagg_sum"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
- checkAnswer(sql(s"select * from maintable_preagg_avg"),
- Seq(Row(1, 31, 1), Row(2, 27, 1), Row(3, 70, 2), Row(4, 55,2)))
- checkAnswer(sql(s"select * from maintable_preagg_count"),
- Seq(Row(1, 1), Row(2, 1), Row(3, 2), Row(4, 2)))
- checkAnswer(sql(s"select * from maintable_preagg_min"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 35), Row(4, 26)))
- checkAnswer(sql(s"select * from maintable_preagg_max"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 35), Row(4, 29)))
- sql("drop table if exists maintable")
- }
-
- test("test load into main table with incremental load") {
- sql("drop table if exists maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('dictionary_include'='id')
- """.stripMargin)
- createAllAggregateTables("maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- checkAnswer(sql(s"select * from maintable_preagg_sum"),
- Seq(Row(1, 31),
- Row(2, 27),
- Row(3, 70),
- Row(4, 55),
- Row(1, 31),
- Row(2, 27),
- Row(3, 70),
- Row(4, 55)))
- checkAnswer(sql(s"select * from maintable_preagg_avg"),
- Seq(Row(1, 31, 1),
- Row(2, 27, 1),
- Row(3, 70, 2),
- Row(4, 55, 2),
- Row(1, 31, 1),
- Row(2, 27, 1),
- Row(3, 70, 2),
- Row(4, 55, 2)))
- checkAnswer(sql(s"select * from maintable_preagg_count"),
- Seq(Row(1, 1), Row(2, 1), Row(3, 2), Row(4, 2), Row(1, 1), Row(2, 1), Row(3, 2), Row(4, 2)))
- checkAnswer(sql(s"select * from maintable_preagg_min"),
- Seq(Row(1, 31),
- Row(2, 27),
- Row(3, 35),
- Row(4, 26),
- Row(1, 31),
- Row(2, 27),
- Row(3, 35),
- Row(4, 26)))
- checkAnswer(sql(s"select * from maintable_preagg_max"),
- Seq(Row(1, 31),
- Row(2, 27),
- Row(3, 35),
- Row(4, 29),
- Row(1, 31),
- Row(2, 27),
- Row(3, 35),
- Row(4, 29)))
- }
-
- test("test to check if exception is thrown for direct load on pre-aggregate table") {
- sql("drop table if exists maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('dictionary_include'='id')
- """.stripMargin)
- sql(
- s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id,sum(age) from maintable group by id"""
- .stripMargin)
- assert(intercept[RuntimeException] {
- sql(s"insert into maintable_preagg_sum values(1, 30)")
- }.getMessage.equalsIgnoreCase("Cannot insert/load data directly into pre-aggregate/child table"))
- }
-
- test("test whether all segments are loaded into pre-aggregate table if segments are set on main table") {
- sql("DROP TABLE IF EXISTS maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"insert into maintable values(1, 'xyz', 'bengaluru', 26)")
- sql(s"insert into maintable values(1, 'xyz', 'bengaluru', 26)")
- sql("set carbon.input.segments.default.maintable=0")
- sql(
- s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id, sum(age) from maintable group by id"""
- .stripMargin)
- sqlContext.sparkSession.catalog.clearCache()
- sql("reset")
- checkAnswer(sql("select * from maintable_preagg_sum"), Row(1, 52))
- }
-
- test("test if pre-aagregate is overwritten if main table is inserted with insert overwrite") {
- sql("DROP TABLE IF EXISTS maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(
- s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id, sum(age) from maintable group by id"""
- .stripMargin)
- sql(s"insert into maintable values(1, 'xyz', 'bengaluru', 26)")
- sql(s"insert into maintable values(1, 'xyz', 'bengaluru', 26)")
- sql(s"insert overwrite table maintable values(1, 'xyz', 'delhi', 29)")
- checkAnswer(sql("select * from maintable_preagg_sum"), Row(1, 29))
- }
-
- test("test load in aggregate table with Measure col") {
- val originalBadRecordsAction = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION)
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, "FAIL")
- sql("drop table if exists y ")
- sql("create table y(year int,month int,name string,salary int) stored by 'carbondata'")
- sql("insert into y select 10,11,'babu',12")
- sql("create datamap y1_sum1 on table y using 'preaggregate' as select year,name,sum(salary) from y group by year,name")
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, originalBadRecordsAction)
- }
-
- test("test partition load into main table with pre-aggregate table") {
- sql("DROP TABLE IF EXISTS maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, city string, age int) partitioned by(name string)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- createAllAggregateTables("maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- checkAnswer(sql(s"select * from maintable_preagg_sum"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
- checkAnswer(sql(s"select * from maintable_preagg_avg"),
- Seq(Row(1, 31, 1), Row(2, 27, 1), Row(3, 70, 2), Row(4, 55, 2)))
- checkAnswer(sql(s"select * from maintable_preagg_count"),
- Seq(Row(1, 1), Row(2, 1), Row(3, 2), Row(4, 2)))
- checkAnswer(sql(s"select * from maintable_preagg_min"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 35), Row(4, 26)))
- checkAnswer(sql(s"select * from maintable_preagg_max"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 35), Row(4, 29)))
- sql("drop table if exists maintable")
- }
-
- test("test load into preaggregate table having group by clause") {
- sql("DROP TABLE IF EXISTS maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"insert into maintable values(1, 'xyz', 'bengaluru', 26)")
- sql(s"insert into maintable values(1, 'xyz', 'bengaluru', 26)")
- sql("set carbon.input.segments.default.maintable=0")
- sql(
- s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id, sum(age) from maintable group by id,name"""
- .stripMargin)
- sqlContext.sparkSession.catalog.clearCache()
- sql("reset")
- checkAnswer(sql("select * from maintable_preagg_sum"), Row(1, 52, "xyz"))
- }
-
- test("test pregarregate with spark adaptive execution ") {
- if (Spark2TestQueryExecutor.spark.version.startsWith("2.3")) {
- // enable adaptive execution
- Spark2TestQueryExecutor.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")
- }
- sql("DROP TABLE IF EXISTS maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(
- s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id, sum(age) from maintable group by id,name"""
- .stripMargin)
- sql(s"insert into maintable values(1, 'xyz', 'bengaluru', 20)")
- sql(s"insert into maintable values(1, 'xyz', 'bengaluru', 30)")
-
- checkAnswer(sql("select id, sum(age) from maintable group by id, name"), Row(1, 50))
- sql("drop datamap preagg_sum on table maintable")
- sql("drop table maintable")
- if (Spark2TestQueryExecutor.spark.version.startsWith("2.3")) {
- // disable adaptive execution
- Spark2TestQueryExecutor.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
- }
- }
-
-
-test("check load and select for avg double datatype") {
- sql("drop table if exists maintbl ")
- sql("create table maintbl(year int,month int,name string,salary double) stored by 'carbondata' tblproperties('sort_scope'='Global_sort','table_blocksize'='23','sort_columns'='month,year,name')")
- sql("insert into maintbl select 10,11,'babu',12.89")
- sql("insert into maintbl select 10,11,'babu',12.89")
- sql("create datamap maintbl_double on table maintbl using 'preaggregate' as select name,avg(salary) from maintbl group by name")
- checkAnswer(sql("select name,avg(salary) from maintbl group by name"), Row("babu", 12.89))
-}
-
- test("check load and select for avg int datatype") {
- sql("drop table if exists maintbl ")
- sql("create table maintbl(year int,month int,name string,salary int) stored by 'carbondata' tblproperties('sort_scope'='Global_sort','table_blocksize'='23','sort_columns'='month,year,name')")
- sql("insert into maintbl select 10,11,'babu',12")
- sql("insert into maintbl select 10,11,'babu',12")
- sql("create datamap maintbl_double on table maintbl using 'preaggregate' as select name,avg(salary) from maintbl group by name")
- checkAnswer(sql("select name,avg(salary) from maintbl group by name"), Row("babu", 12.0))
- }
-
- test("check load and select for avg bigint datatype") {
- sql("drop table if exists maintbl ")
- sql("create table maintbl(year int,month int,name string,salary bigint) stored by 'carbondata' tblproperties('sort_scope'='Global_sort','table_blocksize'='23','sort_columns'='month,year,name')")
- sql("insert into maintbl select 10,11,'babu',12")
- sql("insert into maintbl select 10,11,'babu',12")
- sql("create datamap maintbl_double on table maintbl using 'preaggregate' as select name,avg(salary) from maintbl group by name")
- checkAnswer(sql("select name,avg(salary) from maintbl group by name"), Row("babu", 12.0))
- }
-
- test("check load and select for avg short datatype") {
- sql("drop table if exists maintbl ")
- sql("create table maintbl(year int,month int,name string,salary short) stored by 'carbondata' tblproperties('sort_scope'='Global_sort','table_blocksize'='23','sort_columns'='month,year,name')")
- sql("insert into maintbl select 10,11,'babu',12")
- sql("insert into maintbl select 10,11,'babu',12")
- sql("create datamap maintbl_double on table maintbl using 'preaggregate' as select name,avg(salary) from maintbl group by name")
- checkAnswer(sql("select name,avg(salary) from maintbl group by name"), Row("babu", 12.0))
- }
-
- test("check load and select for avg float datatype") {
- sql("drop table if exists maintbl ")
- sql("create table maintbl(year int,month int,name string,salary float) stored by 'carbondata' tblproperties('sort_scope'='Global_sort','table_blocksize'='23','sort_columns'='month,year,name')")
- sql("insert into maintbl select 10,11,'babu',12")
- sql("insert into maintbl select 10,11,'babu',12")
- val rows = sql("select name,avg(salary) from maintbl group by name").collect()
- sql("create datamap maintbl_double on table maintbl using 'preaggregate' as select name,avg(salary) from maintbl group by name")
- checkAnswer(sql("select name,avg(salary) from maintbl group by name"), rows)
- }
-
- test("create datamap with 'if not exists' after load data into mainTable and create datamap") {
- sql("DROP TABLE IF EXISTS maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(
- s"""
- | create datamap preagg_sum
- | on table maintable
- | using 'preaggregate'
- | as select id,sum(age) from maintable
- | group by id
- """.stripMargin)
-
- sql(
- s"""
- | create datamap if not exists preagg_sum
- | on table maintable
- | using 'preaggregate'
- | as select id,sum(age) from maintable
- | group by id
- """.stripMargin)
-
- checkAnswer(sql(s"select * from maintable_preagg_sum"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
- sql("drop table if exists maintable")
- }
-
- test("create datamap with 'if not exists' after create datamap and load data into mainTable") {
- sql("DROP TABLE IF EXISTS maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
-
- sql(
- s"""
- | create datamap preagg_sum
- | on table maintable
- | using 'preaggregate'
- | as select id,sum(age) from maintable
- | group by id
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(
- s"""
- | create datamap if not exists preagg_sum
- | on table maintable
- | using 'preaggregate'
- | as select id,sum(age) from maintable
- | group by id
- """.stripMargin)
-
- checkAnswer(sql(s"select * from maintable_preagg_sum"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
- sql("drop table if exists maintable")
- }
-
- test("create datamap without 'if not exists' after load data into mainTable and create datamap") {
- sql("DROP TABLE IF EXISTS maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(
- s"""
- | create datamap preagg_sum
- | on table maintable
- | using 'preaggregate'
- | as select id,sum(age) from maintable
- | group by id
- """.stripMargin)
-
- val e: Exception = intercept[MalformedDataMapCommandException] {
- sql(
- s"""
- | create datamap preagg_sum
- | on table maintable
- | using 'preaggregate'
- | as select id,sum(age) from maintable
- | group by id
- """.stripMargin)
- }
- assert(e.getMessage.contains("DataMap name 'preagg_sum' already exist"))
- checkAnswer(sql(s"select * from maintable_preagg_sum"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
- sql("drop table if exists maintable")
- }
-
- test("check load and select for avg int datatype and group by") {
- sql("drop table if exists maintable ")
- sql("CREATE TABLE maintable(id int, city string, age int) stored by 'carbondata'")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- val rows = sql("select age,avg(age) from maintable group by age").collect()
- sql("create datamap maintbl_double on table maintable using 'preaggregate' as select avg(age) from maintable group by age")
- checkAnswer(sql("select age,avg(age) from maintable group by age"), rows)
- sql("drop table if exists maintable ")
- }
-
- test("test load into main table with pre-aggregate table: string") {
- sql(
- """
- | CREATE TABLE main_table(
- | id INT,
- | name STRING,
- | city STRING,
- | age STRING)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- createAllAggregateTables("main_table")
-
- sql(s"LOAD DATA LOCAL INPATH '$testData' INTO TABLE main_table")
- checkAnswer(sql(s"SELECT * FROM main_table_preagg_sum"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
- checkAnswer(sql(s"SELECT * FROM main_table_preagg_avg"),
- Seq(Row(1, 31, 1), Row(2, 27, 1), Row(3, 70, 2), Row(4, 55, 2)))
- checkAnswer(sql(s"SELECT * FROM main_table_preagg_count"),
- Seq(Row(1, 1), Row(2, 1), Row(3, 2), Row(4, 2)))
- checkAnswer(sql(s"SELECT * FROM main_table_preagg_min"),
- Seq(Row(1, "31"), Row(2, "27"), Row(3, "35"), Row(4, "26")))
- checkAnswer(sql(s"SELECT * FROM main_table_preagg_max"),
- Seq(Row(1, "31"), Row(2, "27"), Row(3, "35"), Row(4, "29")))
-
- // check select and match or not match pre-aggregate table
- checkPreAggTable(sql("SELECT id, SUM(age) FROM main_table GROUP BY id"),
- true, "main_table_preagg_sum")
- checkPreAggTable(sql("SELECT id, SUM(age) FROM main_table GROUP BY id"),
- false, "main_table_preagg_avg", "main_table")
-
- checkPreAggTable(sql("SELECT id, AVG(age) FROM main_table GROUP BY id"),
- true, "main_table_preagg_avg")
- checkPreAggTable(sql("SELECT id, AVG(age) from main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- checkPreAggTable(sql("SELECT id, COUNT(age) FROM main_table GROUP BY id"),
- true, "main_table_preagg_count")
- checkPreAggTable(sql("SELECT id, COUNT(age) FROM main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- checkPreAggTable(sql("SELECT id, MIN(age) FROM main_table GROUP BY id"),
- true, "main_table_preagg_min")
- checkPreAggTable(sql("SELECT id, MIN(age) FROM main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- checkPreAggTable(sql("SELECT id, MAX(age) FROM main_table GROUP BY id"),
- true, "main_table_preagg_max")
- checkPreAggTable(sql("SELECT id, MAX(age) FROM main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- // sub query should match pre-aggregate table
- checkPreAggTable(sql("SELECT SUM(age) FROM main_table"),
- true, "main_table_preagg_sum")
- checkPreAggTable(sql("SELECT SUM(age) FROM main_table"),
- false, "main_table_preagg_avg", "main_table")
-
- checkPreAggTable(sql("SELECT AVG(age) FROM main_table GROUP BY id"),
- true, "main_table_preagg_avg")
- checkPreAggTable(sql("SELECT AVG(age) from main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- checkPreAggTable(sql("SELECT COUNT(age) FROM main_table GROUP BY id"),
- true, "main_table_preagg_count")
- checkPreAggTable(sql("SELECT COUNT(age) FROM main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- checkPreAggTable(sql("SELECT MIN(age) FROM main_table GROUP BY id"),
- true, "main_table_preagg_min")
- checkPreAggTable(sql("SELECT MIN(age) FROM main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- checkPreAggTable(sql("SELECT MAX(age) FROM main_table GROUP BY id"),
- true, "main_table_preagg_max")
- checkPreAggTable(sql("SELECT MAX(age) FROM main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
- }
-
- test("test load into main table with pre-aggregate table: sum string column") {
- sql(
- """
- | CREATE TABLE main_table(
- | id INT,
- | name STRING,
- | city STRING,
- | age STRING)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- createAllAggregateTables("main_table", "name")
- sql(s"LOAD DATA LOCAL INPATH '$testData' INTO TABLE main_table")
-
- checkAnswer(sql(s"SELECT * FROM main_table_preagg_sum"),
- Seq(Row(1, null), Row(2, null), Row(3, null), Row(4, null)))
- checkAnswer(sql(s"SELECT * FROM main_table_preagg_avg"),
- Seq(Row(1, null, 1.0), Row(2, null, 1.0), Row(3, null, 2.0), Row(4, null, 2.0)))
- checkAnswer(sql(s"SELECT * FROM main_table_preagg_count"),
- Seq(Row(1, 1), Row(2, 1), Row(3, 2), Row(4, 2)))
- checkAnswer(sql(s"SELECT * FROM main_table_preagg_min"),
- Seq(Row(1, "david"), Row(2, "eason"), Row(3, "jarry"), Row(4, "kunal")))
- checkAnswer(sql(s"SELECT * FROM main_table_preagg_max"),
- Seq(Row(1, "david"), Row(2, "eason"), Row(3, "jarry"), Row(4, "vishal")))
-
- // check select and match or not match pre-aggregate table
- checkPreAggTable(sql("SELECT id, SUM(name) FROM main_table GROUP BY id"),
- true, "main_table_preagg_sum")
- checkPreAggTable(sql("SELECT id, SUM(name) FROM main_table GROUP BY id"),
- false, "main_table_preagg_avg", "main_table")
-
- checkPreAggTable(sql("SELECT id, AVG(name) FROM main_table GROUP BY id"),
- true, "main_table_preagg_avg")
- checkPreAggTable(sql("SELECT id, AVG(name) from main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- checkPreAggTable(sql("SELECT id, COUNT(name) FROM main_table GROUP BY id"),
- true, "main_table_preagg_count")
- checkPreAggTable(sql("SELECT id, COUNT(name) FROM main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- checkPreAggTable(sql("SELECT id, MIN(name) FROM main_table GROUP BY id"),
- true, "main_table_preagg_min")
- checkPreAggTable(sql("SELECT id, MIN(name) FROM main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- checkPreAggTable(sql("SELECT id, MAX(name) FROM main_table GROUP BY id"),
- true, "main_table_preagg_max")
- checkPreAggTable(sql("SELECT id, MAX(name) FROM main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- // sub query should match pre-aggregate table
- checkPreAggTable(sql("SELECT SUM(name) FROM main_table"),
- true, "main_table_preagg_sum")
- checkPreAggTable(sql("SELECT SUM(name) FROM main_table"),
- false, "main_table_preagg_avg", "main_table")
-
- checkPreAggTable(sql("SELECT AVG(name) FROM main_table GROUP BY id"),
- true, "main_table_preagg_avg")
- checkPreAggTable(sql("SELECT AVG(name) from main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- checkPreAggTable(sql("SELECT COUNT(name) FROM main_table GROUP BY id"),
- true, "main_table_preagg_count")
- checkPreAggTable(sql("SELECT COUNT(name) FROM main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- checkPreAggTable(sql("SELECT MIN(name) FROM main_table GROUP BY id"),
- true, "main_table_preagg_min")
- checkPreAggTable(sql("SELECT MIN(name) FROM main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
-
- checkPreAggTable(sql("SELECT MAX(name) FROM main_table GROUP BY id"),
- true, "main_table_preagg_max")
- checkPreAggTable(sql("SELECT MAX(name) FROM main_table GROUP BY id"),
- false, "main_table_preagg_sum", "main_table")
- }
-
- test("test whether all segments are loaded into pre-aggregate table if segments are set on main table 2") {
- sql("DROP TABLE IF EXISTS segmaintable")
- sql(
- """
- | CREATE TABLE segmaintable(
- | id INT,
- | name STRING,
- | city STRING,
- | age INT)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
-
- sql("set carbon.input.segments.default.segmaintable=0")
- checkAnswer(sql(s"SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- Seq(Row(1, 26)))
- sql(
- s"""
- | CREATE DATAMAP preagg_sum
- | ON TABLE segmaintable
- | USING 'preaggregate'
- | AS SELECT id, SUM(age)
- | FROM segmaintable
- | GROUP BY id
- """.stripMargin)
- checkPreAggTable(sql("SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- false, "segmaintable_preagg_sum")
- sqlContext.sparkSession.catalog.clearCache()
- sql("reset")
- checkAnswer(sql("SELECT * FROM segmaintable_preagg_sum"), Seq(Row(1, 26)))
- checkPreAggTable(sql("SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- true, "segmaintable_preagg_sum")
- }
-
- test("test whether all segments are loaded into pre-aggregate table if segments are set on main table 3") {
- sql("DROP TABLE IF EXISTS segmaintable")
- sql(
- """
- | CREATE TABLE segmaintable(
- | id INT,
- | name STRING,
- | city STRING,
- | age INT)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
-
- sql("set carbon.input.segments.default.segmaintable=0")
- checkAnswer(sql(s"SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- Seq(Row(1, 26)))
- sql(
- s"""
- | CREATE DATAMAP preagg_sum
- | ON TABLE segmaintable
- | USING 'preaggregate'
- | AS SELECT id, SUM(age)
- | FROM segmaintable
- | GROUP BY id
- """.stripMargin)
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
- sqlContext.sparkSession.catalog.clearCache()
- sql("reset")
- checkAnswer(sql("SELECT * FROM segmaintable_preagg_sum"), Seq(Row(1, 26), Row(1, 26)))
- checkPreAggTable(sql("SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- true, "segmaintable_preagg_sum")
- }
-
- test("test whether all segments are loaded into pre-aggregate table if segments are set on main table 4") {
- sql("DROP TABLE IF EXISTS segmaintable")
- sql(
- """
- | CREATE TABLE segmaintable(
- | id INT,
- | name STRING,
- | city STRING,
- | age INT)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
-
- // check value before set segments
- checkAnswer(sql(s"SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- Seq(Row(1, 52)))
-
- sql("set carbon.input.segments.default.segmaintable=0")
- // check value after set segments
- checkAnswer(sql(s"SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- Seq(Row(1, 26)))
-
- sql(
- s"""
- | CREATE DATAMAP preagg_sum
- | ON TABLE segmaintable
- | USING 'preaggregate'
- | AS SELECT id, SUM(age)
- | FROM segmaintable
- | GROUP BY id
- """.stripMargin)
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
-
- checkAnswer(sql("SELECT * FROM segmaintable_preagg_sum"), Seq(Row(1, 52), Row(1, 26)))
- checkAnswer(sql(s"SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- Seq(Row(1, 26)))
- checkPreAggTable(sql("SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- false, "segmaintable_preagg_sum")
- sqlContext.sparkSession.catalog.clearCache()
- // reset
- sql("reset")
- checkAnswer(sql(s"SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- Seq(Row(1, 78)))
- checkPreAggTable(sql("SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- true, "segmaintable_preagg_sum")
- }
-
- test("test whether all segments are loaded into pre-aggregate table: auto merge and input segment") {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
- sqlContext.sparkSession.catalog.clearCache()
- sql("reset")
- sql("DROP TABLE IF EXISTS segmaintable")
- sql(
- """
- | CREATE TABLE segmaintable(
- | id INT,
- | name STRING,
- | city STRING,
- | age INT)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
- sql("set carbon.input.segments.default.segmaintable=0")
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
-
- // check value before auto merge
- checkAnswer(sql(s"SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- Seq(Row(1, 26)))
-
-
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
-
- // check value after set segments and auto merge
- checkAnswer(sql(s"SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- Seq.empty)
-
- checkPreAggTable(sql("SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- false, "segmaintable_preagg_sum")
-
- sql(
- s"""
- | CREATE DATAMAP preagg_sum
- | ON TABLE segmaintable
- | USING 'preaggregate'
- | AS SELECT id, SUM(age)
- | FROM segmaintable
- | GROUP BY id
- """.stripMargin)
-
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
- sqlContext.sparkSession.catalog.clearCache()
- // reset
- sql("reset")
- checkAnswer(sql(s"SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- Seq(Row(1, 130)))
- checkPreAggTable(sql("SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- true, "segmaintable_preagg_sum")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
- }
-
- //TODO: need to check and fix
- ignore("test whether all segments are loaded into pre-aggregate table: auto merge and no input segment") {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
- sqlContext.sparkSession.catalog.clearCache()
- sql("reset")
- sql("DROP TABLE IF EXISTS segmaintable")
- sql(
- """
- | CREATE TABLE segmaintable(
- | id INT,
- | name STRING,
- | city STRING,
- | age INT)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
-
- sql(
- s"""
- | CREATE DATAMAP preagg_sum
- | ON TABLE segmaintable
- | USING 'preaggregate'
- | AS SELECT id, SUM(age)
- | FROM segmaintable
- | GROUP BY id
- """.stripMargin)
-
- // check value before auto merge
- checkAnswer(sql(s"SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- Seq(Row(1, 78)))
- checkPreAggTable(sql("SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- true, "segmaintable_preagg_sum")
-
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
-
- // check value after auto merge
- checkPreAggTable(sql("SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- true, "segmaintable_preagg_sum")
-
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
-
- checkAnswer(sql(s"SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- Seq(Row(1, 130)))
- checkPreAggTable(sql("SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- true, "segmaintable_preagg_sum")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
- }
-
- test("test whether all segments are loaded into pre-aggregate table: create after auto merge and no input segment") {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
- sqlContext.sparkSession.catalog.clearCache()
- sql("reset")
- sql("DROP TABLE IF EXISTS segmaintable")
- sql(
- """
- | CREATE TABLE segmaintable(
- | id INT,
- | name STRING,
- | city STRING,
- | age INT)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
- sql(
- s"""
- | CREATE DATAMAP preagg_sum
- | ON TABLE segmaintable
- | USING 'preaggregate'
- | AS SELECT id, SUM(age)
- | FROM segmaintable
- | GROUP BY id
- """.stripMargin)
-
- checkAnswer(sql(s"SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- Seq(Row(1, 130)))
- checkPreAggTable(sql("SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- true, "segmaintable_preagg_sum")
-
- sql(s"INSERT INTO segmaintable VALUES(1, 'xyz', 'bengaluru', 26)")
- checkAnswer(sql(s"SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- Seq(Row(1, 156)))
- checkPreAggTable(sql("SELECT id, SUM(age) FROM segmaintable GROUP BY id"),
- true, "segmaintable_preagg_sum")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
- }
-
- //TODO: need to check and fix
- ignore("test whether all segments are loaded into pre-aggregate table: mixed, load, auto merge and input segment") {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
- sqlContext.sparkSession.catalog.clearCache()
- sql("reset")
- sql("DROP TABLE IF EXISTS main_table")
- sql(
- """
- | CREATE TABLE main_table(
- | id INT,
- | name STRING,
- | city STRING,
- | age INT)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"INSERT INTO main_table VALUES(1, 'xyz', 'bengaluru', 26)")
- sql(s"INSERT INTO main_table VALUES(1, 'xyz', 'bengaluru', 26)")
-
- sql(s"LOAD DATA LOCAL INPATH '$testData' INTO TABLE main_table")
-
- createAllAggregateTables("main_table", "age")
- sql("set carbon.input.segments.default.main_table=0")
-
- checkPreAggTable(sql("SELECT id, SUM(age) FROM main_table GROUP BY id"),
- false, "main_table_preagg_sum")
- checkAnswer(sql(s"SELECT id, SUM(age) FROM main_table GROUP BY id"),
- Seq(Row(1, 26)))
- sqlContext.sparkSession.catalog.clearCache()
- sql("reset")
- checkPreAggTable(sql("SELECT id, SUM(age) FROM main_table GROUP BY id"),
- true, "main_table_preagg_sum")
-
- sql(s"INSERT INTO main_table VALUES(1, 'xyz', 'bengaluru', 26)")
- sql(s"LOAD DATA LOCAL INPATH '$testData' INTO TABLE main_table")
- sql(s"LOAD DATA LOCAL INPATH '$testData' INTO TABLE main_table")
-
- checkAnswer(sql(s"SELECT id, SUM(age) FROM main_table GROUP BY id"),
- Seq(Row(1, 171), Row(2, 81), Row(3, 210), Row(4, 165)))
- checkPreAggTable(sql("SELECT id, SUM(age) FROM main_table GROUP BY id"),
- true, "main_table_preagg_sum")
-
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
- }
-
- //TODO: need to check and fix
- ignore("test whether all segments are loaded into pre-aggregate table: auto merge and check pre-aggregate segment") {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
- sqlContext.sparkSession.catalog.clearCache()
- sql("reset")
- sql("DROP TABLE IF EXISTS main_table")
- sql(
- """
- | CREATE TABLE main_table(
- | id INT,
- | name STRING,
- | city STRING,
- | age INT)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
-
- sql(s"INSERT INTO main_table VALUES(1, 'xyz', 'bengaluru', 26)")
-
- sql(
- s"""
- | CREATE DATAMAP preagg_sum
- | ON TABLE main_table
- | USING 'preaggregate'
- | AS SELECT id, SUM(age)
- | FROM main_table
- | GROUP BY id
- """.stripMargin)
-
- sql(s"INSERT INTO main_table VALUES(1, 'xyz', 'bengaluru', 26)")
- sql(s"LOAD DATA LOCAL INPATH '$testData' INTO TABLE main_table")
-
- checkExistence(sql("show segments for table main_table_preagg_sum"), false, "Compacted")
- sql(s"INSERT INTO main_table VALUES(1, 'xyz', 'bengaluru', 26)")
-
- // check the data whether auto merge
- checkAnswer(sql(s"SELECT id, SUM(age) FROM main_table GROUP BY id"),
- Seq(Row(1, 109), Row(2, 27), Row(3, 70), Row(4, 55)))
- checkExistence(sql("show segments for table main_table_preagg_sum"), true, "Compacted")
-
- sql(s"LOAD DATA LOCAL INPATH '$testData' INTO TABLE main_table")
- sql(s"LOAD DATA LOCAL INPATH '$testData' INTO TABLE main_table")
-
- checkAnswer(sql(s"SELECT id, SUM(age) FROM main_table GROUP BY id"),
- Seq(Row(1, 171), Row(2, 81), Row(3, 210), Row(4, 165)))
- checkPreAggTable(sql("SELECT id, SUM(age) FROM main_table GROUP BY id"),
- true, "main_table_preagg_sum")
-
- checkExistence(sql("show segments for table main_table_preagg_sum"), true, "Compacted")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "false")
- }
-
- test("test deferred rebuild is not supported for preagg") {
- val baseTable = "maintable"
- val preagg = "preaggtable"
- sql(s"DROP TABLE IF EXISTS $baseTable")
- sql(
- s"""
- | CREATE TABLE $baseTable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- val deferredRebuildException = intercept[MalformedDataMapCommandException] {
- sql(
- s"""
- | CREATE DATAMAP $preagg ON TABLE $baseTable
- | USING 'preaggregate'
- | WITH DEFERRED REBUILD
- | AS select id, sum(age) from $baseTable group by id
- """.stripMargin)
- }
- assert(deferredRebuildException.getMessage.contains(
- s"DEFERRED REBUILD is not supported on this datamap $preagg with provider preaggregate"))
-
- sql(
- s"""
- | CREATE DATAMAP $preagg ON TABLE $baseTable
- | USING 'preaggregate'
- | AS select id, sum(age) from $baseTable group by id
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table $baseTable")
- checkExistence(sql(s"SHOW DATAMAP ON TABLE $baseTable"), true, preagg, "preaggregate")
- val exception = intercept[MalformedDataMapCommandException] {
- sql(s"REBUILD DATAMAP $preagg ON TABLE $baseTable").show()
- }
- LOGGER.error(s"XU ${exception.getMessage}")
- assert(exception.getMessage.contains(s"Non-lazy datamap $preagg does not support rebuild"))
- sql(s"DROP TABLE IF EXISTS $baseTable")
- }
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateMisc.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateMisc.scala
deleted file mode 100644
index 55c06c0..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateMisc.scala
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.integration.spark.testsuite.preaggregate
-
-import org.apache.spark.sql.{CarbonEnv, Row}
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.{BeforeAndAfterAll, Ignore}
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-
-@Ignore
-class TestPreAggregateMisc extends QueryTest with BeforeAndAfterAll {
- override def beforeAll: Unit = {
- sql("drop table if exists mainTable")
- sql("CREATE TABLE mainTable(id int, name string, city string, age string) STORED BY 'org.apache.carbondata.format'")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table mainTable")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table mainTable")
- }
-
- test("test PreAggregate With Set Segments property") {
- sql("create datamap agg1 on table mainTable using 'preaggregate' as select name,sum(age) from mainTable group by name")
- sql("SET carbon.input.segments.default.mainTable=0")
- checkAnswer(
- sql("select sum(age) from mainTable"),
- Seq(Row(183.0)))
- sqlContext.sparkSession.catalog.clearCache()
- sql("RESET")
- sql("drop datamap agg1 on table mainTable")
-
- }
-
- test("check preagg tbl properties sort columns inherit from main tbl") {
- sql("drop table if exists y ")
- sql(
- "create table y(year int,month int,name string,salary int) stored by 'carbondata' " +
- "tblproperties('NO_INVERTED_INDEX'='name','sort_scope'='Global_sort'," +
- "'table_blocksize'='23','Dictionary_include'='month','Dictionary_exclude'='year,name'," +
- "'sort_columns'='month,year,name')")
- sql("insert into y select 10,11,'babu',12")
- sql(
- "create datamap y1_sum1 on table y using 'preaggregate' as select year,month,name,sum" +
- "(salary) from y group by year,month,name")
-
- val carbonTable = CarbonEnv.getCarbonTable(Some("default"), "y")(sqlContext.sparkSession)
- val datamaptable = CarbonEnv
- .getCarbonTable(Some("default"), "y_y1_sum1")(sqlContext.sparkSession)
-
- val sortcolumns = datamaptable.getTableInfo.getFactTable.getTableProperties
- .get(CarbonCommonConstants.SORT_COLUMNS)
- val sortcolummatch = sortcolumns != null && sortcolumns.equals("y_month,y_year,y_name")
-
- val sortscope = datamaptable.getTableInfo.getFactTable.getTableProperties.get("sort_scope")
- val sortscopematch = sortscope != null && sortscope.equals(
- carbonTable.getTableInfo.getFactTable.getTableProperties.get("sort_scope"))
- val blockSize = datamaptable.getTableInfo.getFactTable.getTableProperties
- .get(CarbonCommonConstants.TABLE_BLOCKSIZE)
- val blocksizematch = blockSize != null &&
- blockSize.equals(carbonTable.getTableInfo.getFactTable.getTableProperties.
- get(CarbonCommonConstants.TABLE_BLOCKSIZE))
- assert(sortcolummatch && sortscopematch && blocksizematch)
- }
-
- override def afterAll: Unit = {
- sql("DROP TABLE IF EXISTS mainTable")
- sql("DROP TABLE IF EXISTS y")
- }
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
deleted file mode 100644
index 2d2bde1..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
+++ /dev/null
@@ -1,466 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.integration.spark.testsuite.preaggregate
-
-import java.io.File
-
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.sql.hive.CarbonRelation
-import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.TIMESERIES
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.util.SparkQueryTest
-
-class TestPreAggregateTableSelection extends SparkQueryTest with BeforeAndAfterAll {
-
- val timeSeries = TIMESERIES.toString
-
- override def beforeAll: Unit = {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true")
- sql("drop table if exists mainTable")
- sql("drop table if exists mainTableavg")
- sql("drop table if exists agg0")
- sql("drop table if exists agg1")
- sql("drop table if exists agg2")
- sql("drop table if exists agg3")
- sql("drop table if exists agg4")
- sql("drop table if exists agg5")
- sql("drop table if exists agg6")
- sql("drop table if exists agg7")
- sql("DROP TABLE IF EXISTS maintabledict")
- sql("CREATE TABLE mainTable(id int, name string, city string, age string) STORED BY 'org.apache.carbondata.format'")
- sql("create datamap agg0 on table mainTable using 'preaggregate' as select name from mainTable group by name")
- sql("create datamap agg1 on table mainTable using 'preaggregate' as select name,sum(age) from mainTable group by name")
- sql("create datamap agg2 on table mainTable using 'preaggregate' as select name,sum(id) from mainTable group by name")
- sql("create datamap agg3 on table mainTable using 'preaggregate' as select name,count(id) from mainTable group by name")
- sql("create datamap agg4 on table mainTable using 'preaggregate' as select name,sum(age),count(id) from mainTable group by name")
- sql("create datamap agg5 on table mainTable using 'preaggregate' as select name,avg(age) from mainTable group by name")
- sql("create datamap agg6 on table mainTable using 'preaggregate' as select name,min(age) from mainTable group by name")
- sql("create datamap agg7 on table mainTable using 'preaggregate' as select name,max(age) from mainTable group by name")
- sql("create datamap agg8 on table maintable using 'preaggregate' as select name, sum(id), avg(id) from maintable group by name")
- sql("create datamap agg9 on table maintable using 'preaggregate' as select name, count(*) from maintable group by name")
- sql("CREATE TABLE mainTableavg(id int, name string, city string, age bigint) STORED BY 'org.apache.carbondata.format'")
- sql("create datamap agg0 on table mainTableavg using 'preaggregate' as select name,sum(age), avg(age) from mainTableavg group by name")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table mainTable")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table mainTableavg")
- }
-
- test("test PreAggregate table selection 1") {
- val df = sql("select name from mainTable group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0")
- }
-
- test("test PreAggregate table selection with count(*)") {
- val df = sql("select name, count(*) from mainTable group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg9")
- }
-
- test("test PreAggregate table selection 2") {
- val df = sql("select name from mainTable where name in (select name from mainTable) group by name")
- preAggTableValidator(df.queryExecution.analyzed, "mainTable")
- }
-
- test("test PreAggregate table selection 3") {
- val df = sql("select name from mainTable where name in (select name from mainTable group by name) group by name")
- preAggTableValidator(df.queryExecution.analyzed, "mainTable")
- }
-
- test("test PreAggregate table selection 4") {
- val df = sql("select name from mainTable where name in('vishal') group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0")
- }
-
- test("test PreAggregate table selection 5") {
- val df = sql("select name, sum(age) from mainTable group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg1")
- }
-
- test("test PreAggregate table selection with table alias") {
- val df = sql("select name, sum(age) from mainTable as t1 group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg1")
- }
-
- test("test PreAggregate table selection 6") {
- val df = sql("select sum(age) from mainTable group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg1")
- }
-
- test("test PreAggregate table selection 7") {
- val df = sql("select sum(id) from mainTable group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg2")
- }
-
- test("test PreAggregate table selection 8") {
- val df = sql("select count(id) from mainTable group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg3")
- }
-
- test("test PreAggregate table selection 9") {
- val df = sql("select sum(age), count(id) from mainTable group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg4")
- }
-
- test("test PreAggregate table selection 10") {
- val df = sql("select avg(age) from mainTable group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg5")
- }
-
- test("test PreAggregate table selection 11") {
- val df = sql("select max(age) from mainTable group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg7")
- }
-
- test("test PreAggregate table selection 12") {
- val df = sql("select min(age) from mainTable group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg6")
- }
-
- test("test PreAggregate table selection 13") {
- val df = sql("select name, sum(age) from mainTable where city = 'Bangalore' group by name")
- preAggTableValidator(df.queryExecution.analyzed, "mainTable")
- }
-
- test("test PreAggregate table selection 14") {
- val df = sql("select sum(age) from mainTable")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg1")
- }
-
- test("test PreAggregate table selection 15") {
- val df = sql("select avg(age) from mainTable")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg5")
- }
-
- test("test PreAggregate table selection 16") {
- val df = sql("select max(age) from mainTable")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg7")
- }
-
- test("test PreAggregate table selection 17") {
- val df = sql("select min(age) from mainTable")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg6")
- }
-
- test("test PreAggregate table selection 18") {
- val df = sql("select count(id) from mainTable")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg3")
- }
-
- test("test PreAggregate table selection 19: test sum and avg on same column should give proper results") {
- val df = sql("select name, sum(id), avg(id) from maintable group by name")
- checkAnswer(df, Seq(Row("david",1,1.0), Row("jarry",6,3.0), Row("kunal",4,4.0), Row("eason",2,2.0), Row("vishal",4,4.0)))
- checkPreAggTable(df, false, "maintable_agg5", "maintable_agg1")
- checkPreAggTable(df, true, "maintable_agg8")
- }
-
- test("test PreAggregate table selection 20") {
- val df = sql("select name from mainTable group by name order by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0")
- }
-
- test("test PreAggregate table selection 21") {
- val df = sql("select name as NewName from mainTable group by name order by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0")
- }
-
- test("test PreAggregate table selection 22") {
- val df = sql("select name, sum(age) from mainTable group by name order by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg1")
- }
-
- test("test PreAggregate table selection 23") {
- val df = sql("select name as NewName, sum(age) as sum from mainTable group by name order by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg1")
- }
-
- test("test PreAggregate table selection 24") {
- val df = sql("select name as NewName, sum(age) as sum from mainTable where name='vishal' group by name order by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg1")
- }
-
- test("test PreAggregate table selection 25") {
- val df = sql("select name as NewName, sum(age) as sum from mainTable where city = 'Bangalore' group by name order by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable")
- }
-
- test("test PreAggregate table selection 26") {
- val df = sql("select name from mainTable where name='vishal' group by name order by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0")
- }
-
- test("test PreAggregate table selection 27") {
- val df = sql("select name as NewName from mainTable where name='vishal' group by name order by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0")
- }
-
- test("test PreAggregate table selection 28") {
- val df = sql("select name as NewName, sum(case when age=2016 then 1 else 0 end) as sum from mainTable group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable")
- }
-
- def preAggTableValidator(plan: LogicalPlan, actualTableName: String) : Unit ={
- var isValidPlan = false
- plan.transform {
- // first check if any preaTable1 scala function is applied it is present is in plan
- // then call is from create preaTable1regate table class so no need to transform the query plan
- case ca:CarbonRelation =>
- if (ca.isInstanceOf[CarbonDatasourceHadoopRelation]) {
- val relation = ca.asInstanceOf[CarbonDatasourceHadoopRelation]
- if(relation.carbonTable.getTableName.equalsIgnoreCase(actualTableName)) {
- isValidPlan = true
- }
- }
- ca
- case logicalRelation:LogicalRelation =>
- if(logicalRelation.relation.isInstanceOf[CarbonDatasourceHadoopRelation]) {
- val relation = logicalRelation.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
- if(relation.carbonTable.getTableName.equalsIgnoreCase(actualTableName)) {
- isValidPlan = true
- }
- }
- logicalRelation
- }
- if(!isValidPlan) {
- assert(false)
- } else {
- assert(true)
- }
- }
-
-
- test("test PreAggregate table selection 29") {
- val df = sql("select sum(id) from mainTable group by name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg2")
- }
-
- test("test PreAggregate table selection 30") {
- val df = sql("select a.name from mainTable a group by a.name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0")
- }
-
- test("test PreAggregate table selection 31") {
- val df = sql("select a.name as newName from mainTable a group by a.name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0")
- }
-
- test("test PreAggregate table selection 32") {
- val df = sql("select a.name as newName from mainTable a where a.name='vishal' group by a.name")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0")
- }
-
- test("test PreAggregate table selection 33: Test query with math operation hitting fact table") {
- val df = sql("select sum(id)+count(id) from maintable")
- preAggTableValidator(df.queryExecution.analyzed, "maintable")
- }
-
- test("test PreAggregate table selection 34: test if pre-agg table is hit with filter condition") {
- sql("DROP TABLE IF EXISTS filtertable")
- sql(
- """
- | CREATE TABLE filtertable(
- | id INT,
- | name STRING,
- | city STRING,
- | age STRING)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('dictionary_include'='name,age')
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table filtertable")
- sql(
- """
- | CREATE DATAMAP agg9
- | ON TABLE filtertable
- | USING 'preaggregate'
- | AS SELECT name, age, SUM(age)
- | FROM filtertable
- | GROUP BY name, age
- """.stripMargin)
- val df = sql("SELECT name, SUM(age) FROM filtertable WHERE age = '29' GROUP BY name, age")
- preAggTableValidator(df.queryExecution.analyzed, "filtertable_agg9")
- checkAnswer(df, Row("vishal", 29))
- }
-
- test("test PreAggregate table selection 35: test pre-agg table with group by condition") {
- sql("DROP TABLE IF EXISTS grouptable")
- sql(
- """
- | CREATE TABLE grouptable(
- | id INT,
- | name STRING,
- | city STRING,
- | age STRING)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('dictionary_include'='name,age')
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table grouptable")
- sql(
- """
- | CREATE DATAMAP agg9
- | ON TABLE grouptable
- | USING 'preaggregate'
- | AS SELECT SUM(id)
- | FROM grouptable
- | GROUP BY city
- """.stripMargin)
- val df = sql("SELECT SUM(id) FROM grouptable GROUP BY city")
- preAggTableValidator(df.queryExecution.analyzed, "grouptable_agg9")
- checkAnswer(df, Seq(Row(3), Row(3), Row(4), Row(7)))
- }
-
- test("test PreAggregate table selection 36: test PreAggregate table selection with timeseries and normal together") {
- sql("drop table if exists maintabletime")
- sql(
- "create table maintabletime(year int,month int,name string,salary int,dob timestamp) stored" +
- " by 'carbondata' tblproperties('sort_scope'='Global_sort','table_blocksize'='23'," +
- "'sort_columns'='month,year,name')")
- sql("insert into maintabletime select 10,11,'babu',12,'2014-01-01 00:00:00'")
- sql(
- "create datamap agg0 on table maintabletime using 'preaggregate' as select dob,name from " +
- "maintabletime group by dob,name")
-
- sql(
- s"""
- | CREATE DATAMAP agg1_year ON TABLE maintabletime
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dob',
- | 'YEAR_GRANULARITY'='1')
- | AS SELECT dob, name FROM maintabletime
- | GROUP BY dob,name
- """.stripMargin)
-
- val df = sql("SELECT timeseries(dob,'year') FROM maintabletime GROUP BY timeseries(dob,'year')")
- preAggTableValidator(df.queryExecution.analyzed, "maintabletime_agg1_year")
- sql("DROP TABLE IF EXISTS maintabletime")
-
- }
-
- test("test table selection when unsupported aggregate function is present") {
- sql("DROP TABLE IF EXISTS maintabletime")
- sql(
- "create table maintabletime(year int,month int,name string,salary int,dob string) stored" +
- " by 'carbondata' tblproperties('sort_scope'='Global_sort','table_blocksize'='23'," +
- "'sort_columns'='month,year,name')")
- sql("insert into maintabletime select 10,11,'x',12,'2014-01-01 00:00:00'")
- sql(
- "create datamap agg0 on table maintabletime using 'preaggregate' as select name,sum(salary) from " +
- "maintabletime group by name")
-
- sql("select var_samp(name) from maintabletime where name='Mikka' ")
- }
-
- test("test PreAggregate table selection 38: for sum and avg in aggregate table with bigint") {
- val df = sql("select avg(age) from mainTableavg")
- preAggTableValidator(df.queryExecution.analyzed, "mainTableavg_agg0")
- }
-
- test("test PreAggregate table selection for avg with maintable containing dictionary include for group by column") {
- sql(
- "create table maintabledict(year int,month int,name string,salary int,dob string) stored" +
- " by 'carbondata' tblproperties('DICTIONARY_INCLUDE'='year')")
- sql("insert into maintabledict select 10,11,'x',12,'2014-01-01 00:00:00'")
- sql("insert into maintabledict select 10,11,'x',12,'2014-01-01 00:00:00'")
- sql(
- "create datamap aggdict on table maintabledict using 'preaggregate' as select year,avg(year) from " +
- "maintabledict group by year")
- val df = sql("select year,avg(year) from maintabledict group by year")
- checkAnswer(df, Seq(Row(10,10.0)))
- }
-
- test("explain projection query") {
- val rows = sql("explain select name, age from mainTable").collect()
- assertResult(
- """== CarbonData Profiler ==
- |Table Scan on maintable
- | - total: 1 blocks, 1 blocklets
- | - filter: none
- | - pruned by Main DataMap
- | - skipped: 0 blocks, 0 blocklets
- |""".stripMargin)(rows(0).getString(0))
- }
-
- test("explain projection query hit datamap") {
- val rows = sql("explain select name,sum(age) from mainTable group by name").collect()
- assertResult(
- """== CarbonData Profiler ==
- |Query rewrite based on DataMap:
- | - agg1 (preaggregate)
- |Table Scan on maintable_agg1
- | - total: 1 blocks, 1 blocklets
- | - filter: none
- | - pruned by Main DataMap
- | - skipped: 0 blocks, 0 blocklets
- |""".stripMargin)(rows(0).getString(0))
- }
-
- test("explain filter query") {
- sql("explain select name,sum(age) from mainTable where name = 'a' group by name").show(false)
- val rows = sql("explain select name,sum(age) from mainTable where name = 'a' group by name").collect()
- assertResult(
- """== CarbonData Profiler ==
- |Query rewrite based on DataMap:
- | - agg1 (preaggregate)
- |Table Scan on maintable_agg1
- | - total: 1 blocks, 1 blocklets
- | - filter: (maintable_name <> null and maintable_name = a)
- | - pruned by Main DataMap
- | - skipped: 1 blocks, 1 blocklets
- |""".stripMargin)(rows(0).getString(0))
-
- }
-
- test("explain query with multiple table") {
- val query = "explain select t1.city,sum(t1.age) from mainTable t1, mainTableavg t2 " +
- "where t1.name = t2.name and t1.id < 3 group by t1.city"
- sql(query).show(false)
- val rows = sql(query).collect()
- assert(rows(0).getString(0).contains(
- """
- |Table Scan on maintable
- | - total: 1 blocks, 1 blocklets
- | - filter: ((id <> null and id < 3) and name <> null)
- | - pruned by Main DataMap
- | - skipped: 0 blocks, 0 blocklets""".stripMargin))
- assert(rows(0).getString(0).contains(
- """
- |Table Scan on maintableavg
- | - total: 1 blocks, 1 blocklets
- | - filter: name <> null
- | - pruned by Main DataMap
- | - skipped: 0 blocks, 0 blocklets""".stripMargin))
-
- }
-
- override def afterAll: Unit = {
- sql("drop table if exists mainTable")
- sql("drop table if exists mainTable_avg")
- sql("drop table if exists lineitem")
- sql("DROP TABLE IF EXISTS maintabletime")
- sql("DROP TABLE IF EXISTS maintabledict")
- sql("DROP TABLE IF EXISTS mainTableavg")
- sql("DROP TABLE IF EXISTS filtertable")
- sql("DROP TABLE IF EXISTS grouptable")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS,
- CarbonCommonConstants.ENABLE_QUERY_STATISTICS_DEFAULT)
- }
-
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateWithSubQuery.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateWithSubQuery.scala
deleted file mode 100644
index af12be9..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateWithSubQuery.scala
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.integration.spark.testsuite.preaggregate
-
-import org.apache.spark.sql.CarbonDatasourceHadoopRelation
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-
-class TestPreAggregateWithSubQuery extends QueryTest with BeforeAndAfterAll {
-
- override def beforeAll: Unit = {
- sql("DROP TABLE IF EXISTS mainTable")
- sql("DROP TABLE IF EXISTS mainTable1")
- sql("CREATE TABLE mainTable(id int, name string, city string, age string) STORED BY 'org.apache.carbondata.format'")
- sql("CREATE TABLE mainTable1(id int, name string, city string, age string) STORED BY 'org.apache.carbondata.format'")
- sql("create datamap agg0 on table mainTable using 'preaggregate' as select name,sum(age) from mainTable group by name")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table mainTable")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table mainTable1")
- }
-
- test("test sub query PreAggregate table selection 1") {
- val df = sql(
- """
- | SELECT t2.newnewname AS newname
- | FROM mainTable1 t1
- | JOIN (
- | select
- | name AS newnewname,
- | sum(age) AS sum
- | FROM mainTable
- | GROUP BY name ) t2
- | ON t1.name = t2.newnewname
- | GROUP BY t2.newnewname
- """.stripMargin)
- matchTable(collectLogicalRelation(df.queryExecution.analyzed), "maintable_agg0")
- }
-
- test("test sub query PreAggregate table selection 2") {
- val df = sql("select t1.name,t1.city from mainTable1 t1 join (select name as newnewname,sum(age) as sum from mainTable group by name )t2 on t1.name=t2.newnewname")
- matchTable(collectLogicalRelation(df.queryExecution.analyzed), "maintable_agg0")
- }
-
- test("test sub query PreAggregate table selection 3") {
- val df = sql("select t1.name,t2.sum from mainTable1 t1 join (select name as newnewname,sum(age) as sum from mainTable group by name )t2 on t1.name=t2.newnewname")
- matchTable(collectLogicalRelation(df.queryExecution.analyzed), "maintable_agg0")
- }
-
- test("test sub query PreAggregate table selection 4") {
- val df = sql("select t1.name,t2.sum from mainTable1 t1 join (select name,sum(age) as sum from mainTable group by name )t2 on t1.name=t2.name group by t1.name, t2.sum")
- matchTable(collectLogicalRelation(df.queryExecution.analyzed), "maintable_agg0")
- }
-
- /**
- * Below method will be used to collect all the logical relation from logical plan
- * @param logicalPlan
- * query logical plan
- * @return all the logical relation
- */
- def collectLogicalRelation(logicalPlan: LogicalPlan) : Seq[LogicalRelation] = {
- logicalPlan.collect{
- case l:LogicalRelation => l
- }
- }
-
- /**
- * Below method will be used to match the logical relation
- * @param logicalRelations
- * all logical relation
- * @param tableName
- * table name
- */
- def matchTable(logicalRelations: Seq[LogicalRelation], tableName: String) {
- assert(logicalRelations.exists {
- case l:LogicalRelation if l.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
- l.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonTable.getTableName.
- equalsIgnoreCase(tableName)
- })
- }
-
- override def afterAll: Unit = {
- sql("DROP TABLE IF EXISTS mainTable")
- sql("DROP TABLE IF EXISTS mainTable1")
- }
-
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/TestAdaptiveEncodingForPrimitiveTypes.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/TestAdaptiveEncodingForPrimitiveTypes.scala
index 944de37..ca1a4ee 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/TestAdaptiveEncodingForPrimitiveTypes.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/TestAdaptiveEncodingForPrimitiveTypes.scala
@@ -392,22 +392,6 @@ class TestAdaptiveEncodingForPrimitiveTypes extends QueryTest with BeforeAndAfte
sql("drop table if exists negativeTable")
}
- test("test preaggregate datamap on the adaptive encoded column") {
- sql("drop table if exists negativeTable")
- sql("create table negativeTable (intColumn int,stringColumn string,shortColumn short) stored by 'carbondata' TBLPROPERTIES('SORT_COLUMNS'='intColumn,shortColumn')")
- sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' into table negativeTable options('FILEHEADER'='intColumn,stringColumn,shortColumn')")
- // create preagg datamap on the encoded column
- sql("create datamap negativeTable_preAgg1 on table negativeTable USING 'preaggregate' AS select count(intColumn),sum(intColumn) from negativeTable")
- sql("create datamap negativeTable_preAgg2 on table negativeTable USING 'preaggregate' AS select count(shortColumn),sum(shortColumn) from negativeTable")
- checkAnswer(sql("select * from negativeTable"), sql("select * from negativeTable_Compare"))
- checkAnswer(sql("select count(intColumn),sum(intColumn) from negativeTable"),
- Seq(Row(6, 20000)))
- checkAnswer(sql("select count(shortColumn),sum(shortColumn) from negativeTable"),
- Seq(Row(6, 200)))
- sql("drop table if exists negativeTable")
- }
-
-
override def afterAll: Unit = {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT)
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
deleted file mode 100644
index eabe0f5..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
+++ /dev/null
@@ -1,604 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.integration.spark.testsuite.timeseries
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
-
-import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, MalformedDataMapCommandException}
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.TIMESERIES
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach{
-
- val timeSeries = TIMESERIES.toString
- var timestampFormat: String = _
-
- override def beforeAll: Unit = {
- timestampFormat = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- sql("DROP TABLE IF EXISTS mainTable")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- sql("CREATE TABLE mainTable(dataTime timestamp, name string, city string, age int) STORED BY 'org.apache.carbondata.format'")
- sql(
- s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'SECOND_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_hour ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_day ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'day_granularity'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_month ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'month_granularity'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_year ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'year_granularity'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- }
-
- override def afterEach(): Unit = {
- dropDataMaps("mainTable", "agg1_second", "agg1_minute",
- "agg1_hour", "agg1_day", "agg1_month", "agg1_year")
- }
-
- test("test timeseries create table 1") {
- checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_second"), true, "maintable_agg0_second")
- sql("DROP DATAMAP agg0_second ON TABLE mainTable")
- }
-
- test("test timeseries create table 2") {
- checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_hour"), true, "maintable_agg0_hour")
- sql("DROP DATAMAP agg0_hour ON TABLE mainTable")
- }
-
- test("test timeseries create table 3") {
- checkExistence(sql("DESCRIBE FORMATTED maintable_agg0_day"), true, "maintable_agg0_day")
- sql("DROP DATAMAP agg0_day ON TABLE mainTable")
- }
-
- test("test timeseries create table 4") {
- checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_month"), true, "maintable_agg0_month")
- sql("DROP DATAMAP agg0_month ON TABLE mainTable")
- }
-
- test("test timeseries create table 5") {
- checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_year"), true, "maintable_agg0_year")
- sql("DROP DATAMAP agg0_year ON TABLE mainTable")
- }
-
- test("test timeseries create table 6: TIMESERIES should define time granularity") {
- sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
- val e = intercept[MalformedCarbonCommandException] {
- sql(
- s"""CREATE DATAMAP agg0_second ON TABLE mainTable USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'SEC_GRANULARITY'='1')
- |AS SELECT dataTime, SUM(age) FROM mainTable
- |GROUP BY dataTime
- """.stripMargin)
- }
- assert(e.getMessage.contains("TIMESERIES should define time granularity"))
- }
-
- test("test timeseries create table 7: Granularity only support 1") {
- sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
- val e = intercept[MalformedDataMapCommandException] {
- sql(
- s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'DAY_GRANULARITY'='1.5')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- }
- assert(e.getMessage.equals("Granularity only support 1"))
- }
-
- test("test timeseries create table 8: Granularity only support 1") {
- dropDataMaps("mainTable", "agg1_hour")
- val e = intercept[MalformedCarbonCommandException] {
- sql(
- s"""CREATE DATAMAP agg1_hour ON TABLE mainTable USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'HOUR_GRANULARITY'='hour=-2')
- |AS SELECT dataTime, SUM(age) FROM mainTable
- |GROUP BY dataTime
- """.stripMargin)
- }
- assert(e.getMessage.contains("Granularity only support "))
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "maintable_agg1_hour")
- }
-
- test("test timeseries create table 9: SECOND_GRANULARITY is null") {
- sql("DROP DATAMAP IF EXISTS agg1 ON TABLE mainTable")
- val e = intercept[MalformedCarbonCommandException] {
- sql(
- s"""CREATE DATAMAP agg0_hour ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'HOUR_GRANULARITY'='')
- |AS SELECT dataTime, SUM(age) FROM mainTable
- |GROUP BY dataTime
- """.stripMargin)
- }
- assert(e.getMessage.contains("Granularity only support 1"))
- }
-
- test("test timeseries create table 10: Table already exists in database") {
- val e = intercept[MalformedDataMapCommandException] {
- sql(
- s"""CREATE DATAMAP agg1_hour ON TABLE mainTable USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'HOUR_GRANULARITY'='1')
- |AS SELECT dataTime, SUM(age) FROM mainTable
- |GROUP BY dataTime
- """.stripMargin)
- sql(
- s"""CREATE DATAMAP agg1_hour ON TABLE mainTable USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'HOUR_GRANULARITY'='1')
- |AS SELECT dataTime, SUM(age) FROM mainTable
- |GROUP BY dataTime
- """.stripMargin)
- }
- assert(e.getMessage.contains(
- "DataMap name 'agg1_hour' already exist"))
- }
-
- test("test timeseries create table 11: don't support create timeseries table on non timestamp") {
- sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
- val e = intercept[MalformedCarbonCommandException] {
- sql(
- s"""
- | CREATE DATAMAP agg0_hour ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='name',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- }
- assert(e.getMessage.equals("Timeseries event time is only supported on Timestamp column"))
- }
-
- test("test timeseries create table 12: Time series column dataTime does not exists in select") {
- sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
- val e = intercept[MalformedCarbonCommandException] {
- sql(
- s"""
- | CREATE DATAMAP agg0_hour ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT name, SUM(age) FROM mainTable
- | GROUP BY name
- """.stripMargin)
- }
- assert(e.getMessage.equals("Time series column dataTime does not exists in select"))
- }
-
- test("test timeseries create table 13: don't support create timeseries table on non timestamp") {
- sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
- val e = intercept[MalformedCarbonCommandException] {
- sql(
- s"""CREATE DATAMAP agg0_hour ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='name',
- | 'HOUR_GRANULARITY'='1')
- |AS SELECT name, SUM(age) FROM mainTable
- |GROUP BY name
- """.stripMargin)
- }
- assert(e.getMessage.contains("Timeseries event time is only supported on Timestamp column"))
- }
-
- test("test timeseries create table 14: USING") {
- val e: Exception = intercept[MalformedDataMapCommandException] {
- sql(
- """CREATE DATAMAP agg0_hour ON TABLE mainTable
- | USING 'abc'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- }
- assert(e.getMessage.equals("DataMap 'abc' not found"))
- }
-
- test("test timeseries create table 15: USING and catch MalformedCarbonCommandException") {
- val e: Exception = intercept[MalformedCarbonCommandException] {
- sql(
- """CREATE DATAMAP agg0_hour ON TABLE mainTable
- | USING 'abc'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- }
- assert(e.getMessage.equals("DataMap 'abc' not found"))
- }
-
- test("test timeseries create table 16: Only one granularity level can be defined 1") {
- sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
- val e: Exception = intercept[MalformedCarbonCommandException] {
- sql(
- s"""
- | CREATE DATAMAP agg0_hour ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'SECOND_GRANULARITY'='1',
- | 'HOUR_GRANULARITY'='1',
- | 'DAY_GRANULARITY'='1',
- | 'MONTH_GRANULARITY'='1',
- | 'YEAR_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- }
- e.printStackTrace()
- assert(e.getMessage.equals("Only one granularity level can be defined"))
- }
-
- test("test timeseries create table 17: Only one granularity level can be defined 2") {
- sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
- val e: Exception = intercept[MalformedDataMapCommandException] {
- sql(
- s"""
- | CREATE DATAMAP agg0_hour ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'SECOND_GRANULARITY'='1',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- }
- assert(e.getMessage.equals("Only one granularity level can be defined"))
- }
-
- test("test timeseries create table 18: Only one granularity level can be defined 3") {
- sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
- val e: Exception = intercept[MalformedDataMapCommandException] {
- sql(
- s"""
- | CREATE DATAMAP agg0_hour ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'DAY_GRANULARITY'='1',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- }
- assert(e.getMessage.equals("Only one granularity level can be defined"))
- }
-
- test("test timeseries create table 19: timeSeries should define time granularity") {
- sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
- val e = intercept[MalformedDataMapCommandException] {
- sql(
- s"""
- | CREATE DATAMAP agg0_hour ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- }
- assert(e.getMessage.equals(s"$timeSeries should define time granularity"))
- }
-
- test("test timeseries create table 20: should support if not exists, create when same table exists") {
- sql("DROP DATAMAP IF EXISTS agg1 ON TABLE mainTable")
-
- sql(
- s"""
- | CREATE DATAMAP agg1 ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'MONTH_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP IF NOT EXISTS agg1 ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'MONTH_GRANULARITY'='1')
- |AS SELECT dataTime, SUM(age) FROM mainTable
- |GROUP BY dataTime
- """.stripMargin)
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true, "agg1")
- checkExistence(sql("DESC FORMATTED mainTable_agg1"), true, "maintable_age_sum")
- }
-
- test("test timeseries create table 32: should support if not exists, create when same table not exists") {
- sql("DROP DATAMAP IF EXISTS agg1_year ON TABLE mainTable")
- sql(
- s"""
- |CREATE DATAMAP if not exists agg1_year ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'YEAR_GRANULARITY'='1')
- |AS SELECT dataTime, SUM(age) FROM mainTable
- |GROUP BY dataTime
- """.stripMargin)
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true, "agg1_year")
- checkExistence(sql("DESC FORMATTED mainTable_agg1_year"), true, "maintable_age_sum")
- }
-
- test("test timeseries create table 20: don't support 'create datamap if exists'") {
- val e: Exception = intercept[AnalysisException] {
- sql(
- s"""CREATE DATAMAP IF EXISTS agg2 ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime',
- | 'MONTH_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- }
- assert(e.getMessage.contains("identifier matching regex"))
- }
-
- test("test timeseries create table 26: test different data type") {
- sql("drop table if exists dataTable")
- sql(
- s"""
- | CREATE TABLE dataTable(
- | shortField SHORT,
- | booleanField BOOLEAN,
- | intField INT,
- | bigintField LONG,
- | doubleField DOUBLE,
- | stringField STRING,
- | decimalField DECIMAL(18,2),
- | charField CHAR(5),
- | floatField FLOAT,
- | dataTime timestamp
- | )
- | STORED BY 'carbondata'
- """.stripMargin)
-
-
- sql(
- s"""CREATE DATAMAP agg0_hour ON TABLE dataTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT
- | dataTime,
- | SUM(intField),
- | shortField,
- | booleanField,
- | intField,
- | bigintField,
- | doubleField,
- | stringField,
- | decimalField,
- | charField,
- | floatField
- | FROM dataTable
- | GROUP BY
- | dataTime,
- | shortField,
- | booleanField,
- | intField,
- | bigintField,
- | doubleField,
- | stringField,
- | decimalField,
- | charField,
- | floatField
- """.stripMargin)
- checkExistence(sql("SHOW DATAMAP ON TABLE dataTable"), true, "datatable_agg0_hour")
- sql("DROP TABLE IF EXISTS dataTable")
- }
-
- test("test timeseries create table 27: test data map name") {
- sql(
- s"""CREATE DATAMAP agg1_hour ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'HOUR_GRANULARITY'='1')
- |AS SELECT dataTime, SUM(age) FROM mainTable
- |GROUP BY dataTime
- """.stripMargin)
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true, "agg1_hour")
- checkExistence(sql("DESC FORMATTED mainTable_agg1_hour"), true, "maintable_age_sum")
- }
-
- test("test timeseries create table 28: event_time is null") {
- sql("DROP DATAMAP IF EXISTS agg1 ON TABLE mainTable")
- intercept[NullPointerException] {
- sql(
- s"""CREATE DATAMAP agg1 ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='',
- | 'HOUR_GRANULARITY'='1')
- |AS SELECT name, SUM(age) FROM mainTable
- |GROUP BY name
- """.stripMargin)
- }
- }
-
- test("test timeseries create table 29: table not exists") {
- sql("DROP DATAMAP IF EXISTS agg1 ON TABLE mainTable")
- val e = intercept[AnalysisException] {
- sql(
- s"""CREATE DATAMAP agg1 ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'HOUR_GRANULARITY'='1')
- |AS SELECT dataTime, SUM(age) FROM mainTableNo
- |GROUP BY dataTime
- """.stripMargin)
- }
- assert(e.getMessage.contains("Table or view not found: mainTableNo"))
- }
-
- test("test timeseries create table 33: support event_time and granularity key with space") {
- sql("DROP DATAMAP IF EXISTS agg1_month ON TABLE maintable")
- sql(
- s"""CREATE DATAMAP agg1_month ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | ' event_time '='dataTime',
- | ' MONTH_GRANULARITY '='1')
- |AS SELECT dataTime, SUM(age) FROM mainTable
- |GROUP BY dataTime
- """.stripMargin)
- checkExistence(sql("SHOW DATAMAP ON TABLE maintable"), true, "maintable_agg1_month")
- sql("DROP DATAMAP IF EXISTS agg1_month ON TABLE maintable")
- }
-
-
- test("test timeseries create table 34: support event_time value with space") {
- sql("DROP DATAMAP IF EXISTS agg1_month ON TABLE maintable")
- sql(
- s"""CREATE DATAMAP agg1_month ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time '=' dataTime',
- | 'MONTH_GRANULARITY '='1')
- |AS SELECT dataTime, SUM(age) FROM mainTable
- |GROUP BY dataTime
- """.stripMargin)
- checkExistence(sql("SHOW DATAMAP ON TABLE maintable"), true, "maintable_agg1_month")
- sql("DROP DATAMAP IF EXISTS agg1_month ON TABLE maintable")
- }
-
- test("test timeseries create table 35: support granularity value with space") {
- sql("DROP DATAMAP IF EXISTS agg1_month ON TABLE maintable")
- sql(
- s"""CREATE DATAMAP agg1_month ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time '='dataTime',
- | 'MONTH_GRANULARITY '=' 1')
- |AS SELECT dataTime, SUM(age) FROM mainTable
- |GROUP BY dataTime
- """.stripMargin)
- checkExistence(sql("SHOW DATAMAP ON TABLE maintable"), true, "maintable_agg1_month")
- sql("DROP DATAMAP IF EXISTS agg1_month ON TABLE maintable")
- }
-
- test("test timeseries create table 36: support event_time and granularity value with space") {
- sql("DROP DATAMAP IF EXISTS agg1_month ON TABLE maintable")
- sql(
- s"""
- | CREATE DATAMAP agg1_month ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='dataTime ',
- | 'MONTH_GRANULARITY'=' 1 ')
- | AS SELECT dataTime, SUM(age) FROM mainTable
- | GROUP BY dataTime
- """.stripMargin)
- checkExistence(sql("SHOW DATAMAP ON TABLE maintable"), true, "maintable_agg1_month")
- }
-
- test("test timeseries create table 37: unsupport event_time error value") {
- sql("DROP DATAMAP IF EXISTS agg1_month ON TABLE maintable")
- intercept[NullPointerException] {
- sql(
- s"""CREATE DATAMAP agg1_month ON TABLE mainTable USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='data Time',
- | 'MONTH_GRANULARITY'='1')
- |AS SELECT dataTime, SUM(age) FROM mainTable
- |GROUP BY dataTime
- """.stripMargin)
- }
- sql("DROP DATAMAP IF EXISTS agg1_month ON TABLE maintable")
- }
-
- override def afterAll: Unit = {
- dropTable("mainTable")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timestampFormat)
- }
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
deleted file mode 100644
index 33ac413..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.integration.spark.testsuite.timeseries
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
-
-import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, NoSuchDataMapException}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.ProcessMetaDataException
-
-class TestTimeSeriesDropSuite extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
-
- val timeSeries = "timeseries"
- var timestampFormat: String = _
-
- override def beforeAll: Unit = {
- timestampFormat = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- dropTable("mainTable")
- sql(
- """
- | CREATE TABLE mainTable(
- | dataTime TIMESTAMP,
- | name STRING,
- | city STRING,
- | age INT)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- }
-
- override def afterEach(): Unit = {
- dropDataMaps("mainTable", "agg1_second", "agg1_minute",
- "agg1_hour", "agg1_day", "agg1_month", "agg1_year")
- }
-
- test("test timeseries drop datamap 1: drop datamap should throw exception, maintable hasn't datamap") {
- // DROP DATAMAP DataMapName if the DataMapName not exists
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
- val e: Exception = intercept[NoSuchDataMapException] {
- sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
- }
- assert(e.getMessage.equals(
- "Datamap with name agg1_month does not exist"))
- }
-
- test("test timeseries drop datamap 2: should support drop datamap IF EXISTS, maintable hasn't datamap") {
- // DROP DATAMAP IF EXISTS DataMapName
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
- sql(s"DROP DATAMAP IF EXISTS agg1_month ON TABLE mainTable")
- assert(true)
- }
-
- test("test timeseries drop datamap 3: should support drop datamap, maintable has datamap") {
- sql(
- s"""
- | CREATE DATAMAP agg1_month ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'MONTH_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) from mainTable
- | GROUP BY dataTime
- """.stripMargin)
-
- // Before DROP DATAMAP
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true, "agg1_month")
-
- // DROP DATAMAP DataMapName
- sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
- val e: Exception = intercept[NoSuchDataMapException] {
- sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
- }
- assert(e.getMessage.equals(
- "Datamap with name agg1_month does not exist"))
- }
-
- test("test timeseries drop datamap 4: should support drop datamap with IF EXISTS, maintable has datamap") {
- sql(
- s"""
- | CREATE DATAMAP agg1_month ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'MONTH_GRANULARITY'='1')
- | AS SELECT dataTime, SUM(age) from mainTable
- | GROUP BY dataTime
- """.stripMargin)
- // DROP DATAMAP IF EXISTS DataMapName
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true, "agg1_month")
-
- // DROP DATAMAP DataMapName
- sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
- val e: Exception = intercept[MalformedCarbonCommandException] {
- sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
- }
- assert(e.getMessage.equals(
- "Datamap with name agg1_month does not exist"))
- }
-
- test("test timeseries drop datamap 5: drop datamap without IF EXISTS when table not exists, catch MalformedCarbonCommandException") {
- // DROP DATAMAP DataMapName if the DataMapName not exists
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
- val e: Exception = intercept[MalformedCarbonCommandException] {
- sql(s"DROP DATAMAP agg1_month ON TABLE mainTableNotExists")
- }
- assert(e.getMessage.contains(
- "Dropping datamap agg1_month failed: Table or view 'maintablenotexists' not found"))
- }
-
- test("test timeseries drop datamap 6: drop datamap with IF EXISTS when table not exists, catch MalformedCarbonCommandException") {
- // DROP DATAMAP DataMapName if the DataMapName not exists
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
- val e: Exception = intercept[MalformedCarbonCommandException] {
- sql(s"DROP DATAMAP IF EXISTS agg1_month ON TABLE mainTableNotExists")
- }
- assert(e.getMessage.contains(
- "Dropping datamap agg1_month failed: Table or view 'maintablenotexists' not found"))
- }
-
- test("test timeseries drop datamap 7: drop datamap should throw exception if table not exist, catch ProcessMetaDataException") {
- // DROP DATAMAP DataMapName if the DataMapName not exists and
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
- val e: Exception = intercept[ProcessMetaDataException] {
- sql(s"DROP DATAMAP agg1_month ON TABLE mainTableNotExists")
- }
- assert(e.getMessage.contains(
- "Dropping datamap agg1_month failed: Table or view 'maintablenotexists' not found "))
- }
-
- test("test timeseries drop datamap 8: should throw exception if table not exist with IF EXISTS, catch ProcessMetaDataException") {
- // DROP DATAMAP DataMapName if the DataMapName not exists
- // DROP DATAMAP should throw exception if table not exist, even though there is IF EXISTS"
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
- val e: Exception = intercept[ProcessMetaDataException] {
- sql(s"DROP DATAMAP IF EXISTS agg1_month ON TABLE mainTableNotExists")
- }
- assert(e.getMessage.contains(
- "Dropping datamap agg1_month failed: Table or view 'maintablenotexists' not found "))
- }
-
- test("test timeseries drop datamap 9: drop datamap when table not exists, there are datamap in database") {
- sql(
- s"""CREATE DATAMAP agg1_minute ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'minute_GRANULARITY'='1')
- |AS SELECT dataTime, SUM(age) from mainTable
- |GROUP BY dataTime
- """.stripMargin)
-
- // DROP DATAMAP DataMapName if the DataMapName not exists
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
- val e: Exception = intercept[ProcessMetaDataException] {
- sql(s"DROP DATAMAP agg1_month ON TABLE mainTableNotExists")
- }
- assert(e.getMessage.contains(
- "Dropping datamap agg1_month failed: Table or view 'maintablenotexists' not found"))
- }
-
-
- test("test timeseries drop datamap 10: drop datamap when table not exists, there are datamap in database") {
- sql(
- s"""CREATE DATAMAP agg3 ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'month_GRANULARITY'='1')
- |AS SELECT dataTime, SUM(age) from mainTable
- |GROUP BY dataTime
- """.stripMargin)
-
- // DROP DATAMAP DataMapName if the DataMapName not exists
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
- val e: Exception = intercept[ProcessMetaDataException] {
- sql(s"DROP DATAMAP IF EXISTS agg1_month ON TABLE mainTableNotExists")
- }
- assert(e.getMessage.contains(
- "Dropping datamap agg1_month failed: Table or view 'maintablenotexists' not found"))
- }
-
- test("test timeseries drop datamap 11: drop datamap when table not exists, there are datamap in database") {
- sql(
- s"""
- |CREATE DATAMAP agg4 ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='dataTime',
- | 'month_GRANULARITY'='1')
- |AS SELECT dataTime, SUM(age) from mainTable
- |GROUP BY dataTime
- """.stripMargin)
-
- // DROP DATAMAP DataMapName if the DataMapName not exists
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
- val e: Exception = intercept[AnalysisException] {
- sql(s"DROP DATAMAP IF NOT EXISTS agg1_month ON TABLE mainTableNotExists")
- }
- assert(e.getMessage.contains("failure"))
- }
-
- override def afterAll: Unit = {
- dropTable("mainTable")
- dropTable("mainTableNotExists")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timestampFormat)
- }
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesMatchStrategySuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesMatchStrategySuite.scala
deleted file mode 100644
index 55aa264..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesMatchStrategySuite.scala
+++ /dev/null
@@ -1,401 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.integration.spark.testsuite.timeseries
-
-import java.sql.Timestamp
-
-import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.sql.hive.CarbonRelation
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.util.SparkUtil4Test
-import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-class TestTimeSeriesMatchStrategySuite extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
-
- var timestampFormat: String = _
-
- override def beforeAll: Unit = {
- timestampFormat = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- SparkUtil4Test.createTaskMockUp(sqlContext)
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
-
- }
-
- override protected def beforeEach(): Unit = {
- sql("drop table if exists mainTable")
- sql(
- """
- | CREATE TABLE mainTable(
- | mytime TIMESTAMP,
- | name STRING,
- | age INT)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE mainTable")
- }
-
- val timeSeries = "TIMESERIES"
-
- test("test timeseries match 1: select small one when create big_agg and then create small_agg") {
-
- dropDataMaps("maintable", "big_agg", "small_agg")
- sql(
- s"""
- | CREATE DATAMAP big_agg ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age), count(age), max(age), min(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df1 = sql(
- """
- | SELECT
- | timeseries(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE
- | timeseries(mytime,'minute')<'2016-02-23 09:02:00' and
- | timeseries(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | timeseries(mytime,'minute')
- | ORDER BY
- | timeseries(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df1, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60)))
-
- preAggTableValidator(df1.queryExecution.analyzed, "maintable_big_agg")
-
- sql(
- s"""
- | CREATE DATAMAP small_agg ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df2 = sql(
- """
- | SELECT
- | timeseries(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE
- | timeseries(mytime,'minute')<'2016-02-23 09:02:00' and
- | timeseries(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | timeseries(mytime,'minute')
- | ORDER BY
- | timeseries(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df2, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60)))
-
- preAggTableValidator(df2.queryExecution.analyzed, "maintable_small_agg")
- }
-
- test("test timeseries match 2: select small one when create small_agg and then create big_agg") {
- dropDataMaps("maintable", "big_agg", "small_agg")
-
- sql(
- s"""
- | CREATE DATAMAP small_agg ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df2 = sql(
- """
- | SELECT
- | timeseries(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE
- | timeseries(mytime,'minute')<'2016-02-23 09:02:00' and
- | timeseries(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | timeseries(mytime,'minute')
- | ORDER BY
- | timeseries(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df2, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60)))
- preAggTableValidator(df2.queryExecution.analyzed, "maintable_small_agg")
-
- sql(
- s"""
- | CREATE DATAMAP big_agg ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age), count(age), max(age), min(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df1 = sql(
- """
- | SELECT
- | timeseries(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE
- | timeseries(mytime,'minute')<'2016-02-23 09:02:00' and
- | timeseries(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | timeseries(mytime,'minute')
- | ORDER BY
- | timeseries(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df1, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60)))
-
- intercept[Exception]{
- preAggTableValidator(df1.queryExecution.analyzed, "maintable_big_agg")
- }
- preAggTableValidator(df1.queryExecution.analyzed, "maintable_small_agg")
- }
-
- test("test timeseries match 3: select small one when create big_agg and then create small_agg") {
-
- dropDataMaps("maintable", "big_agg", "small_agg")
- sql(
- s"""
- | CREATE DATAMAP big_agg ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age), count(age), max(age), min(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df1 = sql(
- """
- | SELECT
- | timeseries(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE
- | timeseries(mytime,'minute')<'2016-02-23 09:02:00' and
- | timeseries(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | timeseries(mytime,'minute')
- | ORDER BY
- | timeseries(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df1, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60)))
-
- preAggTableValidator(df1.queryExecution.analyzed, "maintable_big_agg")
-
- sql(
- s"""
- | CREATE DATAMAP small_agg ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df2 = sql(
- """
- | SELECT
- | timeseries(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE
- | timeseries(mytime,'minute')<'2016-02-23 09:02:00' and
- | timeseries(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | timeseries(mytime,'minute')
- | ORDER BY
- | timeseries(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df2, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60)))
-
- preAggTableValidator(df2.queryExecution.analyzed, "maintable_small_agg")
- intercept[Exception] {
- preAggTableValidator(df1.queryExecution.analyzed, "maintable_small_agg")
- }
- preAggTableValidator(df1.queryExecution.analyzed, "maintable_big_agg")
- }
-
- test("test timeseries match 4: select small one when create big_agg, small_agg, and middle_agg") {
-
- dropDataMaps("maintable", "big_agg", "small_agg", "middle_agg")
- sql(
- s"""
- | CREATE DATAMAP big_agg ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age), count(age), max(age), min(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df1 = sql(
- """
- | SELECT
- | timeseries(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE
- | timeseries(mytime,'minute')<'2016-02-23 09:02:00' and
- | timeseries(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | timeseries(mytime,'minute')
- | ORDER BY
- | timeseries(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df1, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60)))
-
- preAggTableValidator(df1.queryExecution.analyzed, "maintable_big_agg")
-
- sql(
- s"""
- | CREATE DATAMAP small_agg ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df2 = sql(
- """
- | SELECT
- | timeseries(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE
- | timeseries(mytime,'minute')<'2016-02-23 09:02:00' and
- | timeseries(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | timeseries(mytime,'minute')
- | ORDER BY
- | timeseries(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df2, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60)))
-
- preAggTableValidator(df2.queryExecution.analyzed, "maintable_small_agg")
- intercept[Exception] {
- preAggTableValidator(df1.queryExecution.analyzed, "maintable_small_agg")
- }
- preAggTableValidator(df1.queryExecution.analyzed, "maintable_big_agg")
-
- sql(
- s"""
- | CREATE DATAMAP middle_agg ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age), count(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df3 = sql(
- """
- | SELECT
- | timeseries(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE
- | timeseries(mytime,'minute')<'2016-02-23 09:02:00' and
- | timeseries(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | timeseries(mytime,'minute')
- | ORDER BY
- | timeseries(mytime,'minute')
- """.stripMargin)
-
- preAggTableValidator(df3.queryExecution.analyzed, "maintable_small_agg")
- }
-
- def preAggTableValidator(plan: LogicalPlan, actualTableName: String) : Unit ={
- var isValidPlan = false
- plan.transform {
- // first check if any preaTable1 scala function is applied it is present is in plan
- // then call is FROM create preaTable1regate table class so no need to transform
- // the query plan
- case ca:CarbonRelation =>
- if (ca.isInstanceOf[CarbonDatasourceHadoopRelation]) {
- val relation = ca.asInstanceOf[CarbonDatasourceHadoopRelation]
- if(relation.carbonTable.getTableName.equalsIgnoreCase(actualTableName)) {
- isValidPlan = true
- }
- }
- ca
- case logicalRelation:LogicalRelation =>
- if(logicalRelation.relation.isInstanceOf[CarbonDatasourceHadoopRelation]) {
- val relation = logicalRelation.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
- if(relation.carbonTable.getTableName.equalsIgnoreCase(actualTableName)) {
- isValidPlan = true
- }
- }
- logicalRelation
- }
- if(!isValidPlan) {
- assert(false)
- } else {
- assert(true)
- }
- }
-
- override def afterAll: Unit = {
- dropDataMaps("maintable", "agg0_second", "agg0_hour", "agg0_day", "agg0_month", "agg0_year")
- sql("drop table if exists mainTable")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timestampFormat)
- }
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesUnsupportedSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesUnsupportedSuite.scala
deleted file mode 100644
index 7fa2672..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesUnsupportedSuite.scala
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.integration.spark.testsuite.timeseries
-
-import java.sql.Timestamp
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-class TestTimeSeriesUnsupportedSuite extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
-
- val timeSeries = "TIMESERIES"
- var timestampFormat: String = _
-
- override def beforeAll: Unit = {
- timestampFormat = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- }
-
- override def beforeEach(): Unit = {
- sql("drop table if exists mainTable")
- sql(
- """
- | CREATE TABLE mainTable(
- | mytime TIMESTAMP,
- | name STRING,
- | age INT)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- }
-
- test("test timeseries unsupported 1: don't support insert") {
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE mainTable")
- sql(
- s"""CREATE DATAMAP agg1_minute ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- |AS SELECT mytime, SUM(age) FROM mainTable
- |GROUP BY mytime
- """.stripMargin)
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true, "maintable_agg1_minute")
- checkAnswer(sql("SELECT * FROM maintable_agg1_minute"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00.0"), 60),
- Row(Timestamp.valueOf("2016-02-23 09:02:00.0"), 140)))
- val e = intercept[UnsupportedOperationException] {
- sql(s"INSERT INTO maintable_agg1_minute VALUES('2016-02-23 09:01:00.0', 60)")
- }
- assert(e.getMessage.equalsIgnoreCase(
- "Cannot insert/load data directly into pre-aggregate/child table"))
-
- // check value after inserting
- checkAnswer(sql("SELECT * FROM maintable_agg1_minute"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00.0"), 60),
- Row(Timestamp.valueOf("2016-02-23 09:02:00.0"), 140)))
- }
-
- test("test timeseries unsupported 2: don't support insert") {
- sql(
- s"""CREATE DATAMAP agg1_minute ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- |AS SELECT mytime, SUM(age) FROM mainTable
- |GROUP BY mytime
- """.stripMargin)
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true, "maintable_agg1_minute")
- val e = intercept[UnsupportedOperationException] {
- sql(s"INSERT INTO maintable_agg1_minute VALUES('2016-02-23 09:01:00.0', 60)")
- }
- assert(e.getMessage.equalsIgnoreCase(
- "Cannot insert/load data directly into pre-aggregate/child table"))
- }
-
- test("test timeseries unsupported 3: don't support insert") {
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE mainTable")
- sql(
- s"""CREATE DATAMAP agg1_minute ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- |AS SELECT
- | mytime,
- | name,
- | SUM(age)
- |from mainTable
- |GROUP BY mytime, name
- """.stripMargin)
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true, "maintable_agg1_minute")
-
- val e = intercept[UnsupportedOperationException] {
- sql(s"INSERT INTO maintable_agg1_minute VALUES('2016-02-23 09:01:00.0', 'hello', 60)")
- }
- assert(e.getMessage.equalsIgnoreCase(
- "Cannot insert/load data directly into pre-aggregate/child table"))
- }
-
- test("test timeseries unsupported 4: don't support load") {
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE mainTable")
- sql(
- s"""CREATE DATAMAP agg1_minute ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- |AS SELECT
- | mytime,
- | name,
- | SUM(age) AS age
- |from mainTable
- |GROUP BY mytime, name
- """.stripMargin)
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true, "maintable_agg1_minute")
-
- val e = intercept[UnsupportedOperationException] {
- sql(
- s"""
- | LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv'
- | INTO TABLE maintable_agg1_minute
- | OPTIONS('FILEHEADER'='maintable_mytime,maintable_name,maintable_age_SUM')
- """.stripMargin)
- }
- assert(e.getMessage.equalsIgnoreCase(
- "Cannot insert/load data directly into pre-aggregate/child table"))
- }
-
- test("test timeseries unsupported 5: don't support update") {
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE mainTable")
- sql(
- s"""CREATE DATAMAP agg1_minute ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- |AS SELECT mytime, SUM(age) FROM mainTable
- |GROUP BY mytime
- """.stripMargin)
- checkAnswer(sql("SELECT * FROM maintable_agg1_minute"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00.0"), 60),
- Row(Timestamp.valueOf("2016-02-23 09:02:00.0"), 140)))
- val e = intercept[Exception] {
- sql("update maintable_agg1_minute SET (maintable_age_SUM) = (maintable_age_SUM+1)").show()
- }
- assert(e.getMessage.equalsIgnoreCase(
- "Update operation is not supported for pre-aggregate table"))
-
- // check value after inserting
- checkAnswer(sql("SELECT * FROM maintable_agg1_minute"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00.0"), 60),
- Row(Timestamp.valueOf("2016-02-23 09:02:00.0"), 140)))
- }
-
- test("test timeseries unsupported 6: don't support update") {
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE mainTable")
- sql(
- s"""CREATE DATAMAP agg1_minute ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- |AS SELECT mytime, SUM(age) FROM mainTable
- |GROUP BY mytime
- """.stripMargin)
- checkAnswer(sql("SELECT * FROM maintable_agg1_minute"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00.0"), 60),
- Row(Timestamp.valueOf("2016-02-23 09:02:00.0"), 140)))
- val e = intercept[Exception] {
- sql(
- """
- | update maintable_agg1_minute
- | SET (maintable_mytime, maintable_age_SUM)=('2016-02-23 09:11:00.0', 160)
- | WHERE maintable_age_SUM = '60'
- """.stripMargin).show
- }
- checkAnswer(sql("SELECT * FROM maintable_agg1_minute"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00.0"), 60),
- Row(Timestamp.valueOf("2016-02-23 09:02:00.0"), 140)))
- assert(e.getMessage.equalsIgnoreCase(
- "Update operation is not supported for pre-aggregate table"))
- }
-
- test("test timeseries unsupported 7: don't support delete") {
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE mainTable")
- sql(
- s"""CREATE DATAMAP agg1_minute ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- |AS SELECT mytime, SUM(age) FROM mainTable
- |GROUP BY mytime
- """.stripMargin)
- checkAnswer(sql("SELECT * FROM maintable_agg1_minute"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00.0"), 60),
- Row(Timestamp.valueOf("2016-02-23 09:02:00.0"), 140)))
- val e = intercept[UnsupportedOperationException] {
- sql("delete FROM maintable_agg1_minute")
- }
-
- assert(e.getMessage.equalsIgnoreCase(
- "Delete operation is not supported for pre-aggregate table"))
-
- // check value after inserting
- checkAnswer(sql("SELECT * FROM maintable_agg1_minute"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00.0"), 60),
- Row(Timestamp.valueOf("2016-02-23 09:02:00.0"), 140)))
- }
-
- test("test timeseries unsupported 8: don't support alter") {
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE mainTable")
- sql(
- s"""CREATE DATAMAP agg1_minute ON TABLE mainTable
- |USING '$timeSeries'
- |DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- |AS SELECT mytime, SUM(age) FROM mainTable
- |GROUP BY mytime
- """.stripMargin)
- // before alter
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true, "maintable_agg1_minute")
-
- // alter
- val e = intercept[Exception] {
- sql("alter table maintable_agg1_minute rename to maintable_agg1_minute_new")
- }
- assert(e.getMessage.contains(
- "Rename operation for datamaps is not supported."))
-
- // check datamap after alter
- checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true, "maintable_agg1_minute")
- }
-
- override def afterAll: Unit = {
- dropTable("mainTable")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timestampFormat)
- }
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
deleted file mode 100644
index 2642b03..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.integration.spark.testsuite.timeseries
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.util.SparkUtil4Test
-import org.scalatest.BeforeAndAfterAll
-import org.scalatest.Matchers._
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.TIMESERIES
-import org.apache.carbondata.core.util.CarbonProperties
-
-class TestTimeseriesCompaction extends QueryTest with BeforeAndAfterAll {
-
- var isCompactionEnabled = false
- val timeSeries = TIMESERIES.toString
-
- override def beforeAll: Unit = {
- SparkUtil4Test.createTaskMockUp(sqlContext)
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- isCompactionEnabled = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,"false").toBoolean
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
- sql("drop table if exists mainTable")
- sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
- sql(
- s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'SECOND_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_minute ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_hour ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_day ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'DAY_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_month ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'MONTH_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_year ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'YEAR_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
- }
-
- test("test if pre-agg table is compacted with parent table minor compaction") {
- val segmentNamesSecond = sql("show segments for table maintable_agg0_second").collect().map(_.get(0).toString)
- segmentNamesSecond should equal (Array("3", "2", "1", "0.1", "0"))
-
- val segmentNamesMinute = sql("show segments for table maintable_agg0_minute").collect().map(_.get(0).toString)
- segmentNamesMinute should equal (Array("3", "2", "1", "0.1", "0"))
-
- val segmentNamesHour = sql("show segments for table maintable_agg0_hour").collect().map(_.get(0).toString)
- segmentNamesHour should equal (Array("3", "2", "1", "0.1", "0"))
-
- val segmentNamesday = sql("show segments for table maintable_agg0_day").collect().map(_.get(0).toString)
- segmentNamesday should equal (Array("3", "2", "1", "0.1", "0"))
-
- val segmentNamesmonth = sql("show segments for table maintable_agg0_month").collect().map(_.get(0).toString)
- segmentNamesmonth should equal (Array("3", "2", "1", "0.1", "0"))
-
- val segmentNamesyear = sql("show segments for table maintable_agg0_year").collect().map(_.get(0).toString)
- segmentNamesyear should equal (Array("3", "2", "1", "0.1", "0"))
- }
-
- override def afterAll: Unit = {
- sql("drop table if exists mainTable")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, isCompactionEnabled+"")
- }
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
deleted file mode 100644
index 244cc44..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
+++ /dev/null
@@ -1,597 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.integration.spark.testsuite.timeseries
-
-import java.sql.Timestamp
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.util.SparkUtil4Test
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.TIMESERIES
-import org.apache.carbondata.core.util.CarbonProperties
-
-class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
-
- val timeSeries = TIMESERIES.toString
- var timestampFormat: String = _
-
- override def beforeAll: Unit = {
- timestampFormat = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- SparkUtil4Test.createTaskMockUp(sqlContext)
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- sql("DROP TABLE IF EXISTS mainTable")
- sql("DROP TABLE IF EXISTS table_03")
- sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
- sql(
- s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'SECOND_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_minute ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'minute_granularity'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_hour ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_day ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'DAY_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_month ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'MONTH_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_year ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'year_granularity'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
- sql("CREATE TABLE table_03 (imei string,age int,mac string,productdate timestamp,updatedate timestamp,gamePointId double,contractid double ) STORED BY 'org.apache.carbondata.format'")
- sql(s"LOAD DATA inpath '$resourcesPath/data_sort.csv' INTO table table_03 options ('DELIMITER'=',', 'QUOTECHAR'='','FILEHEADER'='imei,age,mac,productdate,updatedate,gamePointId,contractid')")
-
- sql(
- s"""
- | CREATE DATAMAP ag1_second ON TABLE table_03
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='productdate',
- | 'SECOND_GRANULARITY'='1')
- | AS SELECT productdate,mac,SUM(age) FROM table_03
- | GROUP BY productdate,mac
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP ag1_minute ON TABLE table_03
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='productdate',
- | 'minute_granularity'='1')
- | AS SELECT productdate,mac,SUM(age) FROM table_03
- | GROUP BY productdate,mac
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP ag1_hour ON TABLE table_03
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='productdate',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT productdate,mac,SUM(age) FROM table_03
- | GROUP BY productdate,mac
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP ag1_day ON TABLE table_03
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='productdate',
- | 'DAY_GRANULARITY'='1')
- | AS SELECT productdate,mac,SUM(age) FROM table_03
- | GROUP BY productdate,mac
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP ag1_month ON TABLE table_03
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='productdate',
- | 'month_granularity'='1')
- | AS SELECT productdate,mac,SUM(age) FROM table_03
- | GROUP BY productdate,mac
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP ag1_year ON TABLE table_03
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='productdate',
- | 'year_granularity'='1')
- | AS SELECT productdate,mac,SUM(age) FROM table_03
- | GROUP BY productdate,mac
- """.stripMargin)
-
- }
-
- test("test timeseries table selection 1: year level timeseries data validation1 ") {
- checkAnswer(sql("SELECT COUNT(*) FROM table_03_ag1_year"),
- Seq(Row(4)))
- }
-
- test("test timeseries table selection 2: month level timeseries data validation1 ") {
- checkAnswer(sql("SELECT COUNT(*) FROM table_03_ag1_month"),
- Seq(Row(4)))
- }
-
- test("test timeseries table selection 3: day level timeseries data validation1 ") {
- checkAnswer(sql("SELECT COUNT(*) FROM table_03_ag1_day"),
- Seq(Row(12)))
- }
-
- test("test timeseries table selection 4: year level timeseries data validation") {
- checkAnswer(sql("SELECT * FROM maintable_agg0_year"),
- Seq(Row(Timestamp.valueOf("2016-01-01 00:00:00.0"), 200)))
- }
-
- test("test timeseries table selection 5: month level timeseries data validation") {
- checkAnswer(sql("SELECT * FROM maintable_agg0_month"),
- Seq(Row(Timestamp.valueOf("2016-02-01 00:00:00.0"), 200)))
- }
-
- test("test timeseries table selection 6: day level timeseries data validation") {
- checkAnswer(sql("SELECT * FROM maintable_agg0_day"),
- Seq(Row(Timestamp.valueOf("2016-02-23 00:00:00.0"), 200)))
- }
-
- test("test timeseries table selection 7: hour level timeseries data validation") {
- checkAnswer(sql("SELECT * FROM maintable_agg0_hour"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:00:00.0"), 200)))
- }
-
- test("test timeseries table selection 8: minute level timeseries data validation") {
- checkAnswer(sql("SELECT * FROM maintable_agg0_minute"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00.0"), 60),
- Row(Timestamp.valueOf("2016-02-23 09:02:00.0"), 140)))
- }
-
- test("test timeseries table selection 9: second level timeseries data validation") {
- checkAnswer(sql("SELECT * FROM maintable_agg0_second"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
- Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
- Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
- Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
- Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
- Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
- }
-
- test("test timeseries table selection 10: if timeseries load is successful ON TABLE creation") {
- sql("DROP TABLE IF EXISTS mainTable")
- sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
- sql(
- s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'SECOND_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- checkAnswer( sql("SELECT * FROM maintable_agg0_second"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"),10),
- Row(Timestamp.valueOf("2016-02-23 09:01:40.0"),20),
- Row(Timestamp.valueOf("2016-02-23 09:01:50.0"),30),
- Row(Timestamp.valueOf("2016-02-23 09:02:30.0"),40),
- Row(Timestamp.valueOf("2016-02-23 09:02:40.0"),50),
- Row(Timestamp.valueOf("2016-02-23 09:02:50.0"),50)))
- }
-
- test("test timeseries table selection 11: if timeseries load twice is successful ON TABLE creation") {
- sql("DROP TABLE IF EXISTS mainTable")
- sql(
- """
- | CREATE TABLE mainTable(
- | mytime TIMESTAMP,
- | name STRING,
- | age INT)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE mainTable")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE mainTable")
- sql(
- s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'SECOND_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- checkAnswer(sql("SELECT * FROM maintable_agg0_second"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 20),
- Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 40),
- Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 60),
- Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 80),
- Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 100),
- Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 100)))
- }
-
- test("test timeseries table selection 12: create datamap without 'if not exists' after load data into mainTable and create datamap") {
- sql("DROP TABLE IF EXISTS mainTable")
- sql(
- """
- | CREATE TABLE mainTable(
- | mytime timestamp,
- | name string,
- | age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
-
- sql(
- s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'second_granularity'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- checkAnswer(sql("select * from maintable_agg0_second"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
- Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
- Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
- Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
- Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
- Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
- val e: Exception = intercept[MalformedDataMapCommandException] {
- sql(
- s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'second_granularity'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- }
- assert(e.getMessage.contains("DataMap name 'agg0_second' already exist"))
- sql("DROP DATAMAP agg0_second ON TABLE mainTable")
- }
-
- test("test timeseries table selection 13: create datamap with 'if not exists' after load data into mainTable and create datamap") {
- sql("drop table if exists mainTable")
- sql(
- """
- | CREATE TABLE mainTable(
- | mytime timestamp,
- | name string,
- | age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE mainTable")
-
- sql(
- s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'second_granularity'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- checkAnswer(sql("select * from maintable_agg0_second"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
- Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
- Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
- Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
- Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
- Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
-
- sql(
- s"""
- | CREATE DATAMAP IF NOT EXISTS agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'second_granularity'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- checkAnswer(sql("select * from maintable_agg0_second"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
- Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
- Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
- Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
- Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
- Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
- }
-
- test("test timeseries table selection 14: load data into mainTable after create timeseries datamap ON TABLE and SELECT sub table") {
- sql("DROP TABLE IF EXISTS main_table")
- sql(
- """
- | CREATE TABLE main_table(
- | mytime timestamp,
- | name string,
- | age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
- sql(
- s"""
- | CREATE DATAMAP agg0_second ON TABLE main_table
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'SECOND_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age)
- | FROM main_table
- | GROUP BY mytime""".stripMargin)
-
-
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
-
- checkAnswer(sql("SELECT * FROM main_table_agg0_second"),
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
- Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
- Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
- Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
- Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
- Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50),
- Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
- Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
- Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
- Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
- Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
- Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
- }
-
- test("test timeseries table selection 15: load data into main_table after create timeseries datamap ON TABLE 1") {
- sql("DROP TABLE IF EXISTS main_table")
- sql(
- """
- | CREATE TABLE main_table(
- | mytime timestamp,
- | name string,
- | age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
-
- sql(
- s"""
- | CREATE DATAMAP agg0_minute ON TABLE main_table
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age)
- | FROM main_table
- | GROUP BY mytime""".stripMargin)
-
- sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
- val df = sql(
- """
- | SELECT
- | timeseries(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM main_table
- | WHERE timeseries(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | timeseries(mytime,'minute')
- | ORDER BY
- | timeseries(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df,
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 120),
- Row(Timestamp.valueOf("2016-02-23 09:02:00"), 280)))
- }
-
- test("test timeseries table selection 16: load data into main_table after create timeseries datamap ON TABLE 2") {
- sql("DROP TABLE IF EXISTS main_table")
- sql(
- """
- | CREATE TABLE main_table(
- | mytime timestamp,
- | name string,
- | age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
-
- sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
- sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
- sql(
- s"""
- | CREATE DATAMAP agg0_minute ON TABLE main_table
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age)
- | FROM main_table
- | GROUP BY mytime""".stripMargin)
-
-
- val df = sql(
- """
- | SELECT
- | timeseries(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM main_table
- | WHERE timeseries(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | timeseries(mytime,'minute')
- | ORDER BY
- | timeseries(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df,
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 120),
- Row(Timestamp.valueOf("2016-02-23 09:02:00"), 280)))
- }
-
- test("test timeseries table selection 17: load data into main_table after create timeseries datamap ON TABLE 3") {
- sql("DROP TABLE IF EXISTS main_table")
- sql(
- """
- | CREATE TABLE main_table(
- | mytime timestamp,
- | name string,
- | age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
-
- sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
- sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
- sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
- sql(
- s"""
- | CREATE DATAMAP agg0_minute ON TABLE main_table
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age)
- | FROM main_table
- | GROUP BY mytime""".stripMargin)
-
-
- val df = sql(
- """
- | SELECT
- | timeseries(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM main_table
- | WHERE timeseries(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | timeseries(mytime,'minute')
- | ORDER BY
- | timeseries(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df,
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 180),
- Row(Timestamp.valueOf("2016-02-23 09:02:00"), 420)))
- }
-
- test("test timeseries table selection 18: load data into main_table after create timeseries datamap ON TABLE 4") {
- sql("DROP TABLE IF EXISTS main_table")
- sql(
- """
- | CREATE TABLE main_table(
- | mytime timestamp,
- | name string,
- | age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
-
- sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
-
- sql(
- s"""
- | CREATE DATAMAP agg0_minute ON TABLE main_table
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age)
- | FROM main_table
- | GROUP BY mytime""".stripMargin)
-
- sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
- sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
- val df = sql(
- """
- | SELECT
- | timeseries(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM main_table
- | WHERE timeseries(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | timeseries(mytime,'minute')
- | ORDER BY
- | timeseries(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df,
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 180),
- Row(Timestamp.valueOf("2016-02-23 09:02:00"), 420)))
- }
-
- override def afterAll: Unit = {
- sql("DROP TABLE IF EXISTS main_table")
- sql("DROP TABLE IF EXISTS mainTable")
- sql("DROP TABLE IF EXISTS table_03")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timestampFormat)
- }
-}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
deleted file mode 100644
index 7083c54..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
+++ /dev/null
@@ -1,882 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.integration.spark.testsuite.timeseries
-
-import java.sql.Timestamp
-
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.sql.hive.CarbonRelation
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
-import org.apache.spark.util.SparkUtil4Test
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.TIMESERIES
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-
-class TestTimeseriesTableSelection extends QueryTest with BeforeAndAfterAll {
-
- val timeSeries = TIMESERIES.toString
- var timestampFormat: String = _
-
- override def beforeAll: Unit = {
- timestampFormat = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- SparkUtil4Test.createTaskMockUp(sqlContext)
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- sql("drop table if exists mainTable")
- sql(
- """
- | CREATE TABLE mainTable(mytime timestamp, name string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_second ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'SECOND_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_minute ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'minute_granularity'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_hour ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'HOUR_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_day ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'DAY_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_month ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'MONTH_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP agg0_year ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'EVENT_TIME'='mytime',
- | 'YEAR_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age) FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
- }
-
- test("test timeseries table selection 1") {
- val df = sql("SELECT mytime FROM mainTable GROUP BY mytime")
- preAggTableValidator(df.queryExecution.analyzed, "maintable")
- }
-
- test("test timeseries table selection 2") {
- val df = sql("SELECT TIMESERIES(mytime,'hour') FROM mainTable " +
- "GROUP BY TIMESERIES(mytime,'hour')")
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
- }
-
- test("test timeseries table selection 3: No enum constant MILLI") {
- val e = intercept[Exception] {
- val df = sql(
- """
- | SELECT TIMESERIES(mytime,'milli')
- | FROM mainTable
- | GROUP BY TIMESERIES(mytime,'milli')
- """.stripMargin)
- preAggTableValidator(df.queryExecution.analyzed, "maintable")
- df.show()
- }
- assert(e.getMessage.contains(
- "No enum constant org.apache.carbondata.core.preagg.TimeSeriesFunctionEnum.MILLI"))
- }
-
- test("test timeseries table selection 4") {
- val df = sql("SELECT TIMESERIES(mytime,'year') FROM mainTable GROUP BY TIMESERIES(mytime,'year')")
- preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_year")
- }
-
- test("test timeseries table selection 5") {
- val df = sql("SELECT TIMESERIES(mytime,'day') FROM mainTable GROUP BY TIMESERIES(mytime,'day')")
- preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_day")
- }
-
- test("test timeseries table selection 6") {
- val df = sql("SELECT TIMESERIES(mytime,'month') FROM mainTable GROUP BY TIMESERIES(mytime,'month')")
- preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_month")
- }
-
- test("test timeseries table selection 7") {
- val df = sql("SELECT TIMESERIES(mytime,'minute') FROM mainTable GROUP BY TIMESERIES(mytime,'minute')")
- preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_minute")
- }
-
- test("test timeseries table selection 8") {
- val df = sql("SELECT TIMESERIES(mytime,'second') FROM mainTable GROUP BY TIMESERIES(mytime,'second')")
- preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_second")
- }
-
- test("test timeseries table selection 9") {
- val df = sql(
- """
- | SELECT TIMESERIES(mytime,'hour')
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'hour')='x'
- | GROUP BY TIMESERIES(mytime,'hour')
- """.stripMargin)
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
- }
-
- test("test timeseries table selection 10") {
- val df = sql(
- """
- | SELECT TIMESERIES(mytime,'hour')
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'hour')='x'
- | GROUP BY TIMESERIES(mytime,'hour')
- | ORDER BY TIMESERIES(mytime,'hour')
- """.stripMargin)
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
- }
-
- test("test timeseries table selection 11") {
- val df = sql(
- """
- | SELECT TIMESERIES(mytime,'hour'),SUM(age)
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'hour')='x'
- | GROUP BY TIMESERIES(mytime,'hour')
- | ORDER BY TIMESERIES(mytime,'hour')
- """.stripMargin)
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
- }
-
- test("test timeseries table selection 12") {
- val df = sql(
- """
- | SELECT TIMESERIES(mytime,'hour') AS hourlevel,SUM(age) AS SUM
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'hour')='x'
- | GROUP BY TIMESERIES(mytime,'hour')
- | ORDER BY TIMESERIES(mytime,'hour')
- """.stripMargin)
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
- }
-
- test("test timeseries table selection 13") {
- val df = sql(
- """
- | SELECT TIMESERIES(mytime,'hour')as hourlevel,SUM(age) AS SUM
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'hour')='x' AND name='vishal'
- | GROUP BY TIMESERIES(mytime,'hour')
- | ORDER BY TIMESERIES(mytime,'hour')
- """.stripMargin)
- preAggTableValidator(df.queryExecution.analyzed, "maintable")
- }
-
- test("test timeseries table selection 14: TIMESERIES(mytime,'hour') match") {
- val df = sql(
- """
- | SELECT TIMESERIES(mytime,'hour')
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'hour')='2016-02-23 09:00:00'
- | GROUP BY TIMESERIES(mytime,'hour')
- """.stripMargin)
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
- checkAnswer(df, Row(Timestamp.valueOf("2016-02-23 09:00:00.0")))
- }
-
- test("test timeseries table selection 15: TIMESERIES(mytime,'hour') not match") {
- val df = sql(
- """
- | SELECT TIMESERIES(mytime,'hour')
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'hour')='2016-02-23 09:01:00'
- | GROUP BY TIMESERIES(mytime,'hour')
- """.stripMargin)
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
- checkExistence(df, false, "2016-02-23 09:00:00", "2016-02-23 09:01:00")
- }
-
- test("test timeseries table selection 16: TIMESERIES(mytime,'minute') match") {
- checkExistence(sql("SELECT * FROM mainTable"), true,
- "2016-02-23 09:01:30", "2016-02-23 09:02:40")
- checkExistence(sql("SELECT * FROM mainTable"), false,
- "2016-02-23 09:02:00", "2016-02-23 09:01:00")
- val df = sql(
- """
- |SELECT TIMESERIES(mytime,'minute')
- |FROM mainTable
- |GROUP BY TIMESERIES(mytime,'minute')
- """.stripMargin)
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_minute")
- checkExistence(df, true, "2016-02-23 09:02:00", "2016-02-23 09:01:00")
- checkAnswer(df,
- Seq(Row(Timestamp.valueOf("2016-02-23 09:02:00.0")),
- Row(Timestamp.valueOf("2016-02-23 09:01:00.0"))))
-
- val df2 = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute')as minutelevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')='2016-02-23 09:01:00'
- | GROUP BY TIMESERIES(mytime,'minute')
- | ORDER BY TIMESERIES(mytime,'minute')
- """.stripMargin)
- preAggTableValidator(df2.queryExecution.analyzed, "maintable_agg0_minute")
- checkAnswer(df2, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60)))
- }
-
- test("test timeseries table selection 17: TIMESERIES(mytime,'minute') not match pre agg") {
- val df = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute')as minutelevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')='2016-02-23 09:01:00' AND name='vishal'
- | GROUP BY TIMESERIES(mytime,'minute')
- | ORDER BY TIMESERIES(mytime,'minute')
- """.stripMargin)
- checkAnswer(df, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 10)))
- preAggTableValidator(df.queryExecution.analyzed, "maintable")
- }
-
- test("test timeseries table selection 18: select with many GROUP BY AND one filter") {
- val df = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'year') AS yearLevel,
- | TIMESERIES(mytime,'month') AS monthLevel,
- | TIMESERIES(mytime,'day') AS dayLevel,
- | TIMESERIES(mytime,'hour') AS hourLevel,
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | TIMESERIES(mytime,'second') AS secondLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'year'),
- | TIMESERIES(mytime,'month'),
- | TIMESERIES(mytime,'day'),
- | TIMESERIES(mytime,'hour'),
- | TIMESERIES(mytime,'minute'),
- | TIMESERIES(mytime,'second')
- | ORDER BY
- | TIMESERIES(mytime,'year'),
- | TIMESERIES(mytime,'month'),
- | TIMESERIES(mytime,'day'),
- | TIMESERIES(mytime,'hour'),
- | TIMESERIES(mytime,'minute'),
- | TIMESERIES(mytime,'second')
- """.stripMargin)
-
- checkExistence(df, true,
- "2016-01-01 00:00:00",
- "2016-02-01 00:00:00",
- "2016-02-23 09:00:00",
- "2016-02-23 09:01:00",
- "2016-02-23 09:01:50",
- "30"
- )
- }
-
- test("test timeseries table selection 19: select with many GROUP BY AND many filter") {
- val df = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'year') AS yearLevel,
- | TIMESERIES(mytime,'month') AS monthLevel,
- | TIMESERIES(mytime,'day') AS dayLevel,
- | TIMESERIES(mytime,'hour') AS hourLevel,
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | TIMESERIES(mytime,'second') AS secondLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE
- | TIMESERIES(mytime,'second')='2016-02-23 09:01:50' and
- | TIMESERIES(mytime,'minute')='2016-02-23 09:01:00' and
- | TIMESERIES(mytime,'hour')='2016-02-23 09:00:00' and
- | TIMESERIES(mytime,'month')='2016-02-01 00:00:00' and
- | TIMESERIES(mytime,'year')='2016-01-01 00:00:00'
- | GROUP BY
- | TIMESERIES(mytime,'year'),
- | TIMESERIES(mytime,'month'),
- | TIMESERIES(mytime,'day'),
- | TIMESERIES(mytime,'hour'),
- | TIMESERIES(mytime,'minute'),
- | TIMESERIES(mytime,'second')
- | ORDER BY
- | TIMESERIES(mytime,'year'),
- | TIMESERIES(mytime,'month'),
- | TIMESERIES(mytime,'day'),
- | TIMESERIES(mytime,'hour'),
- | TIMESERIES(mytime,'minute'),
- | TIMESERIES(mytime,'second')
- """.stripMargin)
-
- checkExistence(df, true,
- "2016-01-01 00:00:00",
- "2016-02-01 00:00:00",
- "2016-02-23 09:00:00",
- "2016-02-23 09:01:00",
- "2016-02-23 09:01:50",
- "30"
- )
- }
-
- test("test timeseries table selection 21: filter < AND >") {
- val df = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:04:00'
- | AND TIMESERIES(mytime,'minute')>'2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df, Seq(Row(Timestamp.valueOf("2016-02-23 09:02:00"), 140)))
-
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_minute")
- }
-
- test("test timeseries table selection 22: filter <= AND >=") {
- val df = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<='2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df,
- Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60),
- Row(Timestamp.valueOf("2016-02-23 09:02:00"), 140)))
-
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_minute")
- }
-
- test("test timeseries table selection 23: filter < AND >=") {
- val df = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60)))
-
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_minute")
- }
-
- test("test timeseries table selection 24: filter < AND >=") {
- val df = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:01:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df, Seq.empty)
-
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_minute")
- }
-
- test("test timeseries table selection 25: filter many column") {
- val df = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE
- | TIMESERIES(mytime,'minute')<'2016-02-23 09:02:00' and
- | TIMESERIES(mytime,'hour')>='2016-02-23 09:00:00' and
- | name='vishal'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 10)))
- }
-
- test("test timeseries table selection 26: filter < AND >=, avg") {
- val df = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | avg(age) AS avg
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 20.0)))
-
- }
-
- test("test timeseries table selection 27: filter < AND >=, max") {
- val df = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'second') AS secondLevel,
- | max(age) AS maxValue
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'second')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'second')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'second')
- | ORDER BY
- | TIMESERIES(mytime,'second')
- """.stripMargin)
-
- checkAnswer(df, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30"), 10),
- Row(Timestamp.valueOf("2016-02-23 09:01:40"), 20),
- Row(Timestamp.valueOf("2016-02-23 09:01:50"), 30)))
- }
-
- test("test timeseries table selection 28: filter < AND >=, min") {
- val df = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'second') AS secondLevel,
- | min(age) AS minValue
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'second')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'second')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'second')
- | ORDER BY
- | TIMESERIES(mytime,'second')
- """.stripMargin)
-
- checkAnswer(df, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30"), 10),
- Row(Timestamp.valueOf("2016-02-23 09:01:40"), 20),
- Row(Timestamp.valueOf("2016-02-23 09:01:50"), 30)))
- }
-
- test("test timeseries table selection 29: count, max, min, sum") {
- dropDataMaps("maintable", "agg1_second", "agg1_minute")
- sql(
- s"""
- | CREATE DATAMAP agg1_minute ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age), count(age), max(age), min(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | count(age) AS count
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 3)))
-
- preAggTableValidator(df.queryExecution.analyzed, "maintable_agg1_minute")
-
- val df1 = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | max(age) AS maxValue
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df1, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 30)))
-
- preAggTableValidator(df1.queryExecution.analyzed, "maintable_agg1_minute")
-
- val df2 = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | min(age) AS minValue
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df2, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 10)))
-
- preAggTableValidator(df2.queryExecution.analyzed, "maintable_agg1_minute")
-
- dropDataMaps("maintable", "agg1_second", "agg1_minute")
-
- val df4 = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUM
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df4, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60)))
- }
-
- test("test timeseries table selection 30: max, no create") {
- val df1 = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | max(age) AS maxValue
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df1, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 30)))
- }
-
- test("test timeseries table selection 31: min, no create") {
- val df1 = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | min(age) AS minValue
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df1, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 10)))
- }
-
- test("test timeseries table selection 32: filter < AND >=, min") {
- val df = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'second') AS secondLevel,
- | min(age) AS minValue
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'second')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'second')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'second')
- | ORDER BY
- | TIMESERIES(mytime,'second')
- """.stripMargin)
-
- checkAnswer(df, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30"), 10),
- Row(Timestamp.valueOf("2016-02-23 09:01:40"), 20),
- Row(Timestamp.valueOf("2016-02-23 09:01:50"), 30)))
- }
-
- test("test timeseries table selection 33: max") {
- dropDataMaps("maintable", "agg1_minute")
- sql(
- s"""
- | CREATE DATAMAP agg1_minute ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age), count(age), max(age), min(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df1 = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | max(age) AS maxValue
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df1, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 30)))
-
- preAggTableValidator(df1.queryExecution.analyzed, "maintable_agg1_minute")
- }
-
- test("test timeseries table selection 34: min") {
- dropDataMaps("maintable", "agg1_second", "agg1_minute")
- sql(
- s"""
- | CREATE DATAMAP agg1_minute ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age), count(age), max(age), min(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df1 = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | min(age) AS minValue
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df1, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 10)))
-
- preAggTableValidator(df1.queryExecution.analyzed, "maintable_agg1_minute")
- }
-
- test("test timeseries table selection 35: sum") {
- dropDataMaps("maintable", "agg1_second", "agg1_minute")
- sql(
- s"""
- | CREATE DATAMAP agg1_minute ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age), count(age), max(age), min(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df1 = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | SUM(age) AS SUMValue
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df1, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60)))
- }
-
- test("test timeseries table selection 36: count") {
- dropDataMaps("maintable", "agg1_second", "agg1_minute")
- sql(
- s"""
- | CREATE DATAMAP agg1_minute ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, SUM(age), count(age), max(age), min(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df1 = sql(
- """
- | SELECT
- | TIMESERIES(mytime,'minute') AS minuteLevel,
- | count(age) AS countValue
- | FROM mainTable
- | WHERE TIMESERIES(mytime,'minute')<'2016-02-23 09:02:00'
- | AND TIMESERIES(mytime,'minute')>='2016-02-23 09:01:00'
- | GROUP BY
- | TIMESERIES(mytime,'minute')
- | ORDER BY
- | TIMESERIES(mytime,'minute')
- """.stripMargin)
-
- checkAnswer(df1, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 3)))
-
- preAggTableValidator(df1.queryExecution.analyzed, "maintable_agg1_minute")
-
- }
-
- test("test timeseries table selection 37: avg") {
- dropDataMaps("maintable", "agg1_second", "agg1_minute")
- sql(
- s"""
- | CREATE DATAMAP agg1_minute ON TABLE mainTable
- | USING '$timeSeries'
- | DMPROPERTIES (
- | 'event_time'='mytime',
- | 'MINUTE_GRANULARITY'='1')
- | AS SELECT mytime, avg(age)
- | FROM mainTable
- | GROUP BY mytime
- """.stripMargin)
-
- val df1 = sql(
- """
... 8698 lines suppressed ...