You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/06/22 01:34:02 UTC
[01/50] [abbrv] carbondata git commit: [CARBONDATA-2521] Support
create carbonReader without tableName
Repository: carbondata
Updated Branches:
refs/heads/carbonstore 638ed1fa7 -> b3f782062
[CARBONDATA-2521] Support create carbonReader without tableName
Add new method for creating carbonReader without tableName
1.add new interface: public static CarbonReaderBuilder builder(String tablePath)
2.Default value of table name is UnknownTable + time
This closes #2336
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5b2b9130
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5b2b9130
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5b2b9130
Branch: refs/heads/carbonstore
Commit: 5b2b9130411da7737a76b8901f61c59639113e5d
Parents: b338459
Author: xubo245 <xu...@huawei.com>
Authored: Wed May 23 21:08:23 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Thu May 31 14:19:47 2018 +0800
----------------------------------------------------------------------
docs/sdk-guide.md | 21 +++++-
.../carbondata/sdk/file/CarbonReader.java | 19 +++++
.../carbondata/sdk/file/CarbonReaderTest.java | 76 ++++++++++++++++++++
3 files changed, 113 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5b2b9130/docs/sdk-guide.md
----------------------------------------------------------------------
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index 1d225a9..360516a 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -429,14 +429,29 @@ Find example code at [CarbonReaderExample](https://github.com/apache/carbondata/
### Class org.apache.carbondata.sdk.file.CarbonReader
```
- /**
- * Return a new CarbonReaderBuilder instance
- */
+ /**
+ * Return a new {@link CarbonReaderBuilder} instance
+ *
+ * @param tablePath table store path
+ * @param tableName table name
+ * @return CarbonReaderBuilder object
+ */
public static CarbonReaderBuilder builder(String tablePath, String tableName);
```
```
/**
+ * Return a new CarbonReaderBuilder instance
+ * Default value of table name is table + tablePath + time
+ *
+ * @param tablePath table path
+ * @return CarbonReaderBuilder object
+ */
+ public static CarbonReaderBuilder builder(String tablePath);
+```
+
+```
+ /**
* Return true if has next row
*/
public boolean hasNext();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5b2b9130/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
index 60ead05..81db7b2 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
@@ -18,6 +18,8 @@
package org.apache.carbondata.sdk.file;
import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
import java.util.List;
import org.apache.carbondata.common.annotations.InterfaceAudience;
@@ -83,12 +85,29 @@ public class CarbonReader<T> {
/**
* Return a new {@link CarbonReaderBuilder} instance
+ *
+ * @param tablePath table store path
+ * @param tableName table name
+ * @return CarbonReaderBuilder object
*/
public static CarbonReaderBuilder builder(String tablePath, String tableName) {
return new CarbonReaderBuilder(tablePath, tableName);
}
/**
+ * Return a new {@link CarbonReaderBuilder} instance
+ * Default value of table name is table + tablePath + time
+ *
+ * @param tablePath table path
+ * @return CarbonReaderBuilder object
+ */
+ public static CarbonReaderBuilder builder(String tablePath) {
+ String time = new SimpleDateFormat("yyyyMMddHHmmssSSS").format(new Date());
+ String tableName = "UnknownTable" + time;
+ return builder(tablePath, tableName);
+ }
+
+ /**
* Close reader
*
* @throws IOException
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5b2b9130/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index deb6d06..95c25f8 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -225,6 +225,82 @@ public class CarbonReaderTest extends TestCase {
}
@Test
+ public void testWriteAndReadFilesWithoutTableName() throws IOException, InterruptedException {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[2];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+
+ TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
+
+ CarbonReader reader = CarbonReader
+ .builder(path)
+ .projection(new String[]{"name", "age"})
+ .isTransactionalTable(true)
+ .build();
+
+ // expected output after sorting
+ String[] name = new String[100];
+ int[] age = new int[100];
+ for (int i = 0; i < 100; i++) {
+ name[i] = "robot" + (i / 10);
+ age[i] = (i % 10) * 10 + i / 10;
+ }
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = (Object[]) reader.readNextRow();
+ // Default sort column is applied for dimensions. So, need to validate accordingly
+ Assert.assertEquals(name[i], row[0]);
+ Assert.assertEquals(age[i], row[1]);
+ i++;
+ }
+ Assert.assertEquals(i, 100);
+
+ reader.close();
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ @Test
+ public void testWriteAndReadFilesWithoutTableName2() throws IOException, InterruptedException {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[2];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+
+ TestUtil.writeFilesAndVerify(new Schema(fields), path, true,false);
+
+ CarbonReader reader = CarbonReader
+ .builder(path)
+ .build();
+
+ // expected output after sorting
+ String[] name = new String[100];
+ int[] age = new int[100];
+ for (int i = 0; i < 100; i++) {
+ name[i] = "robot" + (i / 10);
+ age[i] = (i % 10) * 10 + i / 10;
+ }
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = (Object[]) reader.readNextRow();
+ // Default sort column is applied for dimensions. So, need to validate accordingly
+ Assert.assertEquals(name[i], row[0]);
+ Assert.assertEquals(age[i], row[1]);
+ i++;
+ }
+ Assert.assertEquals(i, 100);
+
+ reader.close();
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ @Test
public void testReadSchemaFromDataFile() throws IOException {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
[06/50] [abbrv] carbondata git commit: [CARBONDATA-2566] Optimize
CarbonReaderExample
Posted by ja...@apache.org.
[CARBONDATA-2566] Optimize CarbonReaderExample
Optimize CarbonReaderExample
1.Add different data type, including date and timestamp
2. update the doc
3.invoke the
Schema schema = CarbonSchemaReader
.readSchemaInSchemaFile(dataFiles[0].getAbsolutePath())
.asOriginOrder();
This closes #2356
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/56bf4e42
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/56bf4e42
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/56bf4e42
Branch: refs/heads/carbonstore
Commit: 56bf4e420747ddeb800fc7f004a6ec0d9f5e7d3f
Parents: 9469e6b
Author: xubo245 <xu...@huawei.com>
Authored: Thu May 31 15:52:57 2018 +0800
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Fri Jun 1 16:33:28 2018 +0530
----------------------------------------------------------------------
docs/sdk-guide.md | 15 ++--
.../examples/sdk/CarbonReaderExample.java | 92 +++++++++++++++++---
2 files changed, 89 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56bf4e42/docs/sdk-guide.md
----------------------------------------------------------------------
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index ec70919..2371b33 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -415,17 +415,22 @@ External client can make use of this reader to read CarbonData files without Car
String path = "./testWriteFiles";
CarbonReader reader = CarbonReader
.builder(path, "_temp")
- .projection(new String[]{"name", "age"})
+ .projection(new String[]{"stringField", "shortField", "intField", "longField",
+ "doubleField", "boolField", "dateField", "timeField", "decimalField"})
.build();
// 2. Read data
+ long day = 24L * 3600 * 1000;
int i = 0;
while (reader.hasNext()) {
- Object[] row = (Object[]) reader.readNextRow();
- System.out.println(row[0] + "\t" + row[1]);
- i++;
+ Object[] row = (Object[]) reader.readNextRow();
+ System.out.println(String.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
+ i, row[0], row[1], row[2], row[3], row[4], row[5],
+ new Date((day * ((int) row[6]))), new Timestamp((long) row[7] / 1000), row[8]
+ ));
+ i++;
}
-
+
// 3. Close this reader
reader.close();
```
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56bf4e42/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
index d7886c0..8d3ff0d 100644
--- a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
+++ b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
@@ -18,16 +18,19 @@
package org.apache.carbondata.examples.sdk;
import java.io.File;
+import java.io.FilenameFilter;
+import java.sql.Date;
+import java.sql.Timestamp;
import org.apache.commons.io.FileUtils;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.sdk.file.CarbonReader;
+import org.apache.carbondata.sdk.file.CarbonSchemaReader;
import org.apache.carbondata.sdk.file.CarbonWriter;
import org.apache.carbondata.sdk.file.Field;
import org.apache.carbondata.sdk.file.Schema;
-
/**
* Example fo CarbonReader with close method
* After readNextRow of CarbonReader, User should close the reader,
@@ -39,36 +42,99 @@ public class CarbonReaderExample {
try {
FileUtils.deleteDirectory(new File(path));
- Field[] fields = new Field[2];
- fields[0] = new Field("name", DataTypes.STRING);
- fields[1] = new Field("age", DataTypes.INT);
+ Field[] fields = new Field[9];
+ fields[0] = new Field("stringField", DataTypes.STRING);
+ fields[1] = new Field("shortField", DataTypes.SHORT);
+ fields[2] = new Field("intField", DataTypes.INT);
+ fields[3] = new Field("longField", DataTypes.LONG);
+ fields[4] = new Field("doubleField", DataTypes.DOUBLE);
+ fields[5] = new Field("boolField", DataTypes.BOOLEAN);
+ fields[6] = new Field("dateField", DataTypes.DATE);
+ fields[7] = new Field("timeField", DataTypes.TIMESTAMP);
+ fields[8] = new Field("decimalField", DataTypes.createDecimalType(8, 2));
CarbonWriter writer = CarbonWriter.builder()
- .outputPath(path)
- .persistSchemaFile(true)
- .buildWriterForCSVInput(new Schema(fields));
+ .outputPath(path)
+ .buildWriterForCSVInput(new Schema(fields));
for (int i = 0; i < 10; i++) {
- writer.write(new String[]{"robot" + (i % 10), String.valueOf(i)});
+ String[] row2 = new String[]{
+ "robot" + (i % 10),
+ String.valueOf(i),
+ String.valueOf(i),
+ String.valueOf(Long.MAX_VALUE - i),
+ String.valueOf((double) i / 2),
+ String.valueOf(true),
+ "2019-03-02",
+ "2019-02-12 03:03:34",
+ "12.345"
+ };
+ writer.write(row2);
}
writer.close();
+ File[] dataFiles = new File(path).listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name == null) {
+ return false;
+ }
+ return name.endsWith("carbonindex");
+ }
+ });
+ if (dataFiles == null || dataFiles.length < 1) {
+ throw new RuntimeException("Carbon index file not exists.");
+ }
+ Schema schema = CarbonSchemaReader
+ .readSchemaInIndexFile(dataFiles[0].getAbsolutePath())
+ .asOriginOrder();
+ // Transform the schema
+ String[] strings = new String[schema.getFields().length];
+ for (int i = 0; i < schema.getFields().length; i++) {
+ strings[i] = (schema.getFields())[i].getFieldName();
+ }
+
// Read data
CarbonReader reader = CarbonReader
- .builder(path, "_temp")
- .projection(new String[]{"name", "age"})
- .build();
+ .builder(path, "_temp")
+ .projection(strings)
+ .build();
System.out.println("\nData:");
+ long day = 24L * 3600 * 1000;
+ int i = 0;
while (reader.hasNext()) {
Object[] row = (Object[]) reader.readNextRow();
- System.out.println(row[0] + " " + row[1]);
+ System.out.println(String.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
+ i, row[0], row[1], row[2], row[3], row[4], row[5],
+ new Date((day * ((int) row[6]))), new Timestamp((long) row[7] / 1000), row[8]
+ ));
+ i++;
+ }
+ System.out.println("\nFinished");
+
+ // Read data
+ CarbonReader reader2 = CarbonReader
+ .builder(path, "_temp")
+ .projectAllColumns()
+ .build();
+
+ System.out.println("\nData:");
+ i = 0;
+ while (reader2.hasNext()) {
+ Object[] row = (Object[]) reader2.readNextRow();
+ System.out.println(String.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
+ i, row[0], new Date((day * ((int) row[1]))), new Timestamp((long) row[2] / 1000),
+ row[3], row[4], row[5], row[6], row[7], row[8]
+ ));
+ i++;
}
System.out.println("\nFinished");
reader.close();
FileUtils.deleteDirectory(new File(path));
- } catch (Exception e) {
+ } catch (Throwable e) {
e.printStackTrace();
+ System.out.println(e.getMessage());
}
}
}
[03/50] [abbrv] carbondata git commit: [Documentation] Editorial
Review comment fixed
Posted by ja...@apache.org.
[Documentation] Editorial Review comment fixed
Editorial Review comment fixed
This closes #2320
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5ad70095
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5ad70095
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5ad70095
Branch: refs/heads/carbonstore
Commit: 5ad7009573b7a95a181221d6a58df05e1fafbeb6
Parents: 6aadfe7
Author: sgururajshetty <sg...@gmail.com>
Authored: Thu May 31 17:36:26 2018 +0530
Committer: kunal642 <ku...@gmail.com>
Committed: Thu May 31 17:40:30 2018 +0530
----------------------------------------------------------------------
docs/data-management-on-carbondata.md | 4 ++--
docs/datamap/timeseries-datamap-guide.md | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ad70095/docs/data-management-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/data-management-on-carbondata.md b/docs/data-management-on-carbondata.md
index 51e98ab..706209c 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -35,11 +35,11 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
```
CREATE TABLE [IF NOT EXISTS] [db_name.]table_name[(col_name data_type , ...)]
- STORED BY 'carbondata'
+ STORED AS carbondata
[TBLPROPERTIES (property_name=property_value, ...)]
[LOCATION 'path']
```
- **NOTE:** CarbonData also supports "STORED AS carbondata". Find example code at [CarbonSessionExample](https://github.com/apache/carbondata/blob/master/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala) in the CarbonData repo.
+ **NOTE:** CarbonData also supports "STORED AS carbondata" and "USING carbondata". Find example code at [CarbonSessionExample](https://github.com/apache/carbondata/blob/master/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala) in the CarbonData repo.
### Usage Guidelines
Following are the guidelines for TBLPROPERTIES, CarbonData's additional table options can be set via carbon.properties.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ad70095/docs/datamap/timeseries-datamap-guide.md
----------------------------------------------------------------------
diff --git a/docs/datamap/timeseries-datamap-guide.md b/docs/datamap/timeseries-datamap-guide.md
index 7847312..bea5286 100644
--- a/docs/datamap/timeseries-datamap-guide.md
+++ b/docs/datamap/timeseries-datamap-guide.md
@@ -1,12 +1,12 @@
# CarbonData Timeseries DataMap
-* [Timeseries DataMap](#timeseries-datamap-intoduction-(alpha-feature-in-1.3.0))
+* [Timeseries DataMap Introduction](#timeseries-datamap-intoduction)
* [Compaction](#compacting-pre-aggregate-tables)
* [Data Management](#data-management-with-pre-aggregate-tables)
-## Timeseries DataMap Intoduction (Alpha feature in 1.3.0)
-Timeseries DataMap a pre-aggregate table implementation based on 'preaggregate' DataMap.
-Difference is that Timerseries DataMap has built-in understanding of time hierarchy and
+## Timeseries DataMap Introduction (Alpha feature in 1.3.0)
+Timeseries DataMap a pre-aggregate table implementation based on 'pre-aggregate' DataMap.
+Difference is that Timeseries DataMap has built-in understanding of time hierarchy and
levels: year, month, day, hour, minute, so that it supports automatic roll-up in time dimension
for query.
[19/50] [abbrv] carbondata git commit: [HOTFIX][CARBONDATA-2591] Fix
SDK CarbonReader filter issue
Posted by ja...@apache.org.
[HOTFIX][CARBONDATA-2591] Fix SDK CarbonReader filter issue
There are some issue in SDK CarbonReader filter function, please check the lira.
This closes #2363
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/290ef5a3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/290ef5a3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/290ef5a3
Branch: refs/heads/carbonstore
Commit: 290ef5a3a90081b3c95ea0dc418f643ea5ad694f
Parents: 0ef7e55
Author: xubo245 <xu...@huawei.com>
Authored: Thu Jun 7 22:00:31 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Jun 12 10:54:04 2018 +0530
----------------------------------------------------------------------
.../core/metadata/schema/table/CarbonTable.java | 35 +++
.../apache/carbondata/core/util/CarbonUtil.java | 1 +
.../sdk/file/CarbonReaderBuilder.java | 6 +-
.../carbondata/sdk/file/CarbonReaderTest.java | 251 ++++++++++++++++++-
.../apache/carbondata/sdk/file/TestUtil.java | 14 ++
5 files changed, 304 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/290ef5a3/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 6949643..20bc7a1 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -17,6 +17,8 @@
package org.apache.carbondata.core.metadata.schema.table;
+import java.io.File;
+import java.io.FilenameFilter;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
@@ -57,6 +59,8 @@ import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
+import static org.apache.carbondata.core.util.CarbonUtil.thriftColumnSchemaToWrapperColumnSchema;
+
/**
* Mapping class for Carbon actual table
*/
@@ -218,6 +222,37 @@ public class CarbonTable implements Serializable {
}
}
+ public static CarbonTable buildTable(
+ String tablePath,
+ String tableName) throws IOException {
+ TableInfo tableInfoInfer = CarbonUtil.buildDummyTableInfo(tablePath, "null", "null");
+ File[] dataFiles = new File(tablePath).listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name == null) {
+ return false;
+ }
+ return name.endsWith("carbonindex");
+ }
+ });
+ if (dataFiles == null || dataFiles.length < 1) {
+ throw new RuntimeException("Carbon index file not exists.");
+ }
+ org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil
+ .inferSchemaFromIndexFile(dataFiles[0].toString(), tableName);
+ List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
+ for (org.apache.carbondata.format.ColumnSchema thriftColumnSchema : tableInfo
+ .getFact_table().getTable_columns()) {
+ ColumnSchema columnSchema = thriftColumnSchemaToWrapperColumnSchema(thriftColumnSchema);
+ if (columnSchema.getColumnReferenceId() == null) {
+ columnSchema.setColumnReferenceId(columnSchema.getColumnUniqueId());
+ }
+ columnSchemaList.add(columnSchema);
+ }
+ tableInfoInfer.getFactTable().setListOfColumns(columnSchemaList);
+ return CarbonTable.buildFromTableInfo(tableInfoInfer);
+ }
+
public static CarbonTable buildDummyTable(String tablePath) throws IOException {
TableInfo tableInfoInfer = CarbonUtil.buildDummyTableInfo(tablePath, "null", "null");
return CarbonTable.buildFromTableInfo(tableInfoInfer);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/290ef5a3/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index e1e5e16..2aa4a05 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2209,6 +2209,7 @@ public final class CarbonUtil {
org.apache.carbondata.format.ColumnSchema externalColumnSchema) {
ColumnSchema wrapperColumnSchema = new ColumnSchema();
wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id());
+ wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId());
wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name());
wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar());
DataType dataType = thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/290ef5a3/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
index 98aa6e0..83cb34e 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
@@ -179,7 +179,11 @@ public class CarbonReaderBuilder {
if (isTransactionalTable) {
table = CarbonTable.buildFromTablePath(tableName, "default", tablePath);
} else {
- table = CarbonTable.buildDummyTable(tablePath);
+ if (filterExpression != null) {
+ table = CarbonTable.buildTable(tablePath, tableName);
+ } else {
+ table = CarbonTable.buildDummyTable(tablePath);
+ }
}
final CarbonFileInputFormat format = new CarbonFileInputFormat();
final Job job = new Job(new Configuration());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/290ef5a3/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index a8aa795..fb2e2bc 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -20,8 +20,7 @@ package org.apache.carbondata.sdk.file;
import java.io.*;
import java.sql.Date;
import java.sql.Timestamp;
-import java.util.Arrays;
-import java.util.Comparator;
+import java.util.*;
import org.apache.avro.generic.GenericData;
import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
@@ -29,6 +28,11 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
+import org.apache.carbondata.core.scan.expression.logical.AndExpression;
+import org.apache.carbondata.core.scan.expression.logical.OrExpression;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -48,6 +52,12 @@ public class CarbonReaderTest extends TestCase {
@After
public void verifyDMFile() {
assert (!TestUtil.verifyMdtFile());
+ String path = "./testWriteFiles";
+ try {
+ FileUtils.deleteDirectory(new File(path));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
}
@Test
@@ -106,6 +116,243 @@ public class CarbonReaderTest extends TestCase {
}
@Test
+ public void testReadWithFilterOfTransactional() throws IOException, InterruptedException {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[2];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path, true);
+
+ EqualToExpression equalToExpression = new EqualToExpression(
+ new ColumnExpression("name", DataTypes.STRING),
+ new LiteralExpression("robot1", DataTypes.STRING));
+ CarbonReader reader = CarbonReader
+ .builder(path, "_temp")
+ .isTransactionalTable(true)
+ .projection(new String[]{"name", "age"})
+ .filter(equalToExpression)
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = (Object[]) reader.readNextRow();
+ // Default sort column is applied for dimensions. So, need to validate accordingly
+ assert ("robot1".equals(row[0]));
+ i++;
+ }
+ Assert.assertEquals(i, 20);
+
+ reader.close();
+
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ @Test
+ public void testReadWithFilterOfTransactionalAnd() throws IOException, InterruptedException {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[3];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+ fields[2] = new Field("doubleField", DataTypes.DOUBLE);
+
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path, true);
+
+ ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE);
+ EqualToExpression equalToExpression = new EqualToExpression(columnExpression,
+ new LiteralExpression("3.5", DataTypes.DOUBLE));
+
+ ColumnExpression columnExpression2 = new ColumnExpression("name", DataTypes.STRING);
+ EqualToExpression equalToExpression2 = new EqualToExpression(columnExpression2,
+ new LiteralExpression("robot7", DataTypes.STRING));
+
+ AndExpression andExpression = new AndExpression(equalToExpression, equalToExpression2);
+ CarbonReader reader = CarbonReader
+ .builder(path, "_temp")
+ .isTransactionalTable(true)
+ .projection(new String[]{"name", "age", "doubleField"})
+ .filter(andExpression)
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = (Object[]) reader.readNextRow();
+ assert (((String) row[0]).contains("robot7"));
+ assert (7 == (int) (row[1]));
+ assert (3.5 == (double) (row[2]));
+ i++;
+ }
+ Assert.assertEquals(i, 1);
+
+ reader.close();
+
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ @Test
+ public void testReadWithFilterOfNonTransactionalSimple() throws IOException, InterruptedException {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[2];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false);
+
+ ColumnExpression columnExpression = new ColumnExpression("name", DataTypes.STRING);
+ EqualToExpression equalToExpression = new EqualToExpression(columnExpression,
+ new LiteralExpression("robot1", DataTypes.STRING));
+
+ CarbonReader reader = CarbonReader
+ .builder(path, "_temp")
+ .isTransactionalTable(false)
+ .projection(new String[]{"name", "age"})
+ .filter(equalToExpression)
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = (Object[]) reader.readNextRow();
+ // Default sort column is applied for dimensions. So, need to validate accordingly
+ assert ("robot1".equals(row[0]));
+ i++;
+ }
+ Assert.assertEquals(i, 20);
+
+ reader.close();
+
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ @Test
+ public void testReadWithFilterOfNonTransactional2() throws IOException, InterruptedException {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[2];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false);
+
+ ColumnExpression columnExpression = new ColumnExpression("age", DataTypes.INT);
+
+ EqualToExpression equalToExpression = new EqualToExpression(columnExpression,
+ new LiteralExpression("1", DataTypes.INT));
+ CarbonReader reader = CarbonReader
+ .builder(path, "_temp")
+ .isTransactionalTable(false)
+ .projection(new String[]{"name", "age"})
+ .filter(equalToExpression)
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = (Object[]) reader.readNextRow();
+ // Default sort column is applied for dimensions. So, need to validate accordingly
+ assert (((String) row[0]).contains("robot"));
+ assert (1 == (int) (row[1]));
+ i++;
+ }
+ Assert.assertEquals(i, 1);
+
+ reader.close();
+
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ @Test
+ public void testReadWithFilterOfNonTransactionalAnd() throws IOException, InterruptedException {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[3];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+ fields[2] = new Field("doubleField", DataTypes.DOUBLE);
+
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false);
+
+ ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE);
+ EqualToExpression equalToExpression = new EqualToExpression(columnExpression,
+ new LiteralExpression("3.5", DataTypes.DOUBLE));
+
+ ColumnExpression columnExpression2 = new ColumnExpression("name", DataTypes.STRING);
+ EqualToExpression equalToExpression2 = new EqualToExpression(columnExpression2,
+ new LiteralExpression("robot7", DataTypes.STRING));
+
+ AndExpression andExpression = new AndExpression(equalToExpression, equalToExpression2);
+ CarbonReader reader = CarbonReader
+ .builder(path, "_temp")
+ .isTransactionalTable(false)
+ .projection(new String[]{"name", "age", "doubleField"})
+ .filter(andExpression)
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = (Object[]) reader.readNextRow();
+ assert (((String) row[0]).contains("robot7"));
+ assert (7 == (int) (row[1]));
+ assert (3.5 == (double) (row[2]));
+ i++;
+ }
+ Assert.assertEquals(i, 1);
+
+ reader.close();
+
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ @Test
+ public void testReadWithFilterOfNonTransactionalOr() throws IOException, InterruptedException {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[3];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+ fields[2] = new Field("doubleField", DataTypes.DOUBLE);
+
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false);
+
+ ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE);
+ EqualToExpression equalToExpression = new EqualToExpression(columnExpression,
+ new LiteralExpression("3.5", DataTypes.DOUBLE));
+
+ ColumnExpression columnExpression2 = new ColumnExpression("name", DataTypes.STRING);
+ EqualToExpression equalToExpression2 = new EqualToExpression(columnExpression2,
+ new LiteralExpression("robot7", DataTypes.STRING));
+
+ OrExpression andExpression = new OrExpression(equalToExpression, equalToExpression2);
+ CarbonReader reader = CarbonReader
+ .builder(path, "_temp")
+ .isTransactionalTable(false)
+ .projection(new String[]{"name", "age", "doubleField"})
+ .filter(andExpression)
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = (Object[]) reader.readNextRow();
+ assert (((String) row[0]).contains("robot7"));
+ assert (7 == ((int) (row[1]) % 10));
+ assert (0.5 == ((double) (row[2]) % 1));
+ i++;
+ }
+ Assert.assertEquals(i, 20);
+
+ reader.close();
+
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ @Test
public void testReadColumnTwice() throws IOException, InterruptedException {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/290ef5a3/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java
index 0f00d61..919472c 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java
@@ -49,6 +49,20 @@ public class TestUtil {
}
/**
+ * write file and verify
+ *
+ * @param rows number of rows
+ * @param schema schema
+ * @param path table store path
+ * @param persistSchema whether persist schema
+ * @param isTransactionalTable whether is transactional table
+ */
+ public static void writeFilesAndVerify(int rows, Schema schema, String path, boolean persistSchema,
+ boolean isTransactionalTable) {
+ writeFilesAndVerify(rows, schema, path, null, persistSchema, -1, -1, isTransactionalTable);
+ }
+
+ /**
* Invoke CarbonWriter API to write carbon files and assert the file is rewritten
* @param rows number of rows to write
* @param schema schema of the file
[50/50] [abbrv] carbondata git commit: [CARBONDATA-2623][DataMap] Add
DataMap Pre and Pevent listener
Posted by ja...@apache.org.
[CARBONDATA-2623][DataMap] Add DataMap Pre and Pevent listener
Added Pre and Post Execution Events for index datamap
This closes #2389
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b3f78206
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b3f78206
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b3f78206
Branch: refs/heads/carbonstore
Commit: b3f7820623d4bc9ab4408beb8ad708ba9b19b899
Parents: 55f4bc6
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Wed Jun 20 19:52:51 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Thu Jun 21 17:37:48 2018 +0530
----------------------------------------------------------------------
.../carbondata/events/DataMapEvents.scala | 68 ++++++++++++++++++++
.../org/apache/carbondata/events/Events.scala | 18 +++++-
.../datamap/IndexDataMapRebuildRDD.scala | 11 +++-
.../spark/rdd/CarbonTableCompactor.scala | 23 ++++++-
.../datamap/CarbonCreateDataMapCommand.scala | 22 +++++++
.../datamap/CarbonDataMapRebuildCommand.scala | 12 ++++
.../datamap/CarbonDropDataMapCommand.scala | 11 ++++
.../management/CarbonLoadDataCommand.scala | 21 +++++-
8 files changed, 181 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b3f78206/integration/spark-common/src/main/scala/org/apache/carbondata/events/DataMapEvents.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/events/DataMapEvents.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/events/DataMapEvents.scala
new file mode 100644
index 0000000..8fb374f
--- /dev/null
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/events/DataMapEvents.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.events
+
+import org.apache.spark.sql.SparkSession
+
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+
+/**
+ * For handling operation's after finish of index creation over table with index datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class CreateDataMapPostExecutionEvent(sparkSession: SparkSession,
+ storePath: String) extends Event with CreateDataMapEventsInfo
+
+/**
+ * For handling operation's before start of update index datmap status over table with index datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class UpdateDataMapPreExecutionEvent(sparkSession: SparkSession,
+ storePath: String) extends Event with CreateDataMapEventsInfo
+
+/**
+ * For handling operation's after finish of update index datmap status over table with index
+ * datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class UpdateDataMapPostExecutionEvent(sparkSession: SparkSession,
+ storePath: String) extends Event with CreateDataMapEventsInfo
+
+/**
+ * For handling operation's before start of index build over table with index datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class BuildDataMapPreExecutionEvent(sparkSession: SparkSession,
+ identifier: AbsoluteTableIdentifier, dataMapNames: scala.collection.mutable.Seq[String])
+ extends Event with BuildDataMapEventsInfo
+
+/**
+ * For handling operation's after finish of index build over table with index datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class BuildDataMapPostExecutionEvent(sparkSession: SparkSession,
+ identifier: AbsoluteTableIdentifier)
+ extends Event with TableEventInfo
+
+/**
+ * For handling operation's before start of index creation over table with index datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class CreateDataMapPreExecutionEvent(sparkSession: SparkSession,
+ storePath: String) extends Event with CreateDataMapEventsInfo
+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b3f78206/integration/spark-common/src/main/scala/org/apache/carbondata/events/Events.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/events/Events.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/events/Events.scala
index da62e02..1830a35 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/events/Events.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/events/Events.scala
@@ -21,7 +21,6 @@ import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
import org.apache.spark.sql.execution.command.{AlterTableAddColumnsModel, AlterTableDataTypeChangeModel, AlterTableDropColumnModel, AlterTableRenameModel, CarbonMergerMapping}
-import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
@@ -168,3 +167,20 @@ trait DeleteFromTableEventInfo {
trait SessionEventInfo {
val sparkSession: SparkSession
}
+
+/**
+ * Event info for create datamap
+ */
+trait CreateDataMapEventsInfo {
+ val sparkSession: SparkSession
+ val storePath: String
+}
+
+/**
+ * Event info for build datamap
+ */
+trait BuildDataMapEventsInfo {
+ val sparkSession: SparkSession
+ val identifier: AbsoluteTableIdentifier
+ val dataMapNames: scala.collection.mutable.Seq[String]
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b3f78206/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
index cde6201..d064306 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
@@ -22,6 +22,7 @@ import java.text.SimpleDateFormat
import java.util
import scala.collection.JavaConverters._
+import scala.collection.mutable
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapred.JobConf
@@ -42,6 +43,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.TaskMetricsMap
import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.events.{BuildDataMapPostExecutionEvent, BuildDataMapPreExecutionEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.hadoop.{CarbonInputSplit, CarbonMultiBlockSplit, CarbonProjection, CarbonRecordReader}
import org.apache.carbondata.hadoop.api.{CarbonInputFormat, CarbonTableInputFormat}
import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport
@@ -67,13 +69,20 @@ object IndexDataMapRebuildRDD {
val validAndInvalidSegments = segmentStatusManager.getValidAndInvalidSegments()
val validSegments = validAndInvalidSegments.getValidSegments
val indexedCarbonColumns = carbonTable.getIndexedColumns(schema)
-
+ val operationContext = new OperationContext()
+ val buildDataMapPreExecutionEvent = new BuildDataMapPreExecutionEvent(sparkSession,
+ tableIdentifier,
+ mutable.Seq[String](schema.getDataMapName))
+ OperationListenerBus.getInstance().fireEvent(buildDataMapPreExecutionEvent, operationContext)
// loop all segments to rebuild DataMap
validSegments.asScala.foreach { segment =>
// if lucene datamap folder is exists, not require to build lucene datamap again
refreshOneSegment(sparkSession, carbonTable, schema.getDataMapName,
indexedCarbonColumns, segment.getSegmentNo);
}
+ val buildDataMapPostExecutionEvent = new BuildDataMapPostExecutionEvent(sparkSession,
+ tableIdentifier)
+ OperationListenerBus.getInstance().fireEvent(buildDataMapPostExecutionEvent, operationContext)
}
private def refreshOneSegment(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b3f78206/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
index 7605b9d..fcc649e 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
@@ -22,12 +22,13 @@ import java.util.List
import java.util.concurrent.ExecutorService
import scala.collection.JavaConverters._
+import scala.collection.mutable
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.execution.command.{CarbonMergerMapping, CompactionCallableModel, CompactionModel}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datamap.Segment
+import org.apache.carbondata.core.datamap.{DataMapStoreManager, Segment}
import org.apache.carbondata.core.metadata.SegmentFileStore
import org.apache.carbondata.core.readcommitter.{ReadCommittedScope, TableStatusReadCommittedScope}
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatusManager}
@@ -156,7 +157,18 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
carbonMergerMapping,
mergedLoadName)
OperationListenerBus.getInstance.fireEvent(alterTableCompactionPreEvent, operationContext)
-
+ // Add pre event listener for index datamap
+ val tableDataMaps = DataMapStoreManager.getInstance().getAllDataMap(carbonTable)
+ val dataMapOperationContext = new OperationContext()
+ if (null != tableDataMaps) {
+ val dataMapNames: mutable.Buffer[String] =
+ tableDataMaps.asScala.map(dataMap => dataMap.getDataMapSchema.getDataMapName)
+ val dataMapPreExecutionEvent: BuildDataMapPreExecutionEvent =
+ new BuildDataMapPreExecutionEvent(sqlContext.sparkSession,
+ carbonTable.getAbsoluteTableIdentifier, dataMapNames)
+ OperationListenerBus.getInstance().fireEvent(dataMapPreExecutionEvent,
+ dataMapOperationContext)
+ }
var execInstance = "1"
// in case of non dynamic executor allocation, number of executors are fixed.
if (sc.sparkContext.getConf.contains("spark.executor.instances")) {
@@ -272,6 +284,13 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
mergedLoadName)
OperationListenerBus.getInstance()
.fireEvent(compactionLoadStatusPostEvent, operationContext)
+ if (null != tableDataMaps) {
+ val buildDataMapPostExecutionEvent: BuildDataMapPostExecutionEvent =
+ new BuildDataMapPostExecutionEvent(sqlContext.sparkSession,
+ carbonTable.getAbsoluteTableIdentifier)
+ OperationListenerBus.getInstance()
+ .fireEvent(buildDataMapPostExecutionEvent, dataMapOperationContext)
+ }
val commitDone = operationContext.getProperty("commitComplete")
val commitComplete = if (null != commitDone) {
commitDone.toString.toBoolean
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b3f78206/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index 1ae872a..27e1720 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -31,7 +31,9 @@ import org.apache.carbondata.core.datamap.{DataMapProvider, DataMapStoreManager}
import org.apache.carbondata.core.datamap.status.DataMapStatusManager
import org.apache.carbondata.core.metadata.schema.datamap.{DataMapClassProvider, DataMapProperty}
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema}
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.datamap.{DataMapManager, IndexDataMapProvider}
+import org.apache.carbondata.events._
/**
* Below command class will be used to create datamap on table
@@ -108,8 +110,18 @@ case class CarbonCreateDataMapCommand(
"column '%s' already has datamap created", column.getColName))
}
}
+ val operationContext: OperationContext = new OperationContext()
+ val systemFolderLocation: String = CarbonProperties.getInstance().getSystemFolderLocation
+ val createDataMapPreExecutionEvent: CreateDataMapPreExecutionEvent =
+ new CreateDataMapPreExecutionEvent(sparkSession, systemFolderLocation)
+ OperationListenerBus.getInstance().fireEvent(createDataMapPreExecutionEvent,
+ operationContext)
dataMapProvider.initMeta(queryString.orNull)
DataMapStatusManager.disableDataMap(dataMapName)
+ val createDataMapPostExecutionEvent: CreateDataMapPostExecutionEvent =
+ new CreateDataMapPostExecutionEvent(sparkSession, systemFolderLocation)
+ OperationListenerBus.getInstance().fireEvent(createDataMapPostExecutionEvent,
+ operationContext)
case _ =>
if (deferredRebuild) {
throw new MalformedDataMapCommandException(
@@ -128,7 +140,17 @@ case class CarbonCreateDataMapCommand(
if (mainTable != null && !deferredRebuild) {
dataMapProvider.rebuild()
if (dataMapSchema.isIndexDataMap) {
+ val operationContext: OperationContext = new OperationContext()
+ val systemFolderLocation: String = CarbonProperties.getInstance().getSystemFolderLocation
+ val updateDataMapPreExecutionEvent: UpdateDataMapPreExecutionEvent =
+ new UpdateDataMapPreExecutionEvent(sparkSession, systemFolderLocation)
+ OperationListenerBus.getInstance().fireEvent(updateDataMapPreExecutionEvent,
+ operationContext)
DataMapStatusManager.enableDataMap(dataMapName)
+ val updateDataMapPostExecutionEvent: UpdateDataMapPostExecutionEvent =
+ new UpdateDataMapPostExecutionEvent(sparkSession, systemFolderLocation)
+ OperationListenerBus.getInstance().fireEvent(updateDataMapPostExecutionEvent,
+ operationContext)
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b3f78206/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapRebuildCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapRebuildCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapRebuildCommand.scala
index 6493c83..beadc7e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapRebuildCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapRebuildCommand.scala
@@ -23,7 +23,9 @@ import org.apache.spark.sql.execution.command.DataCommand
import org.apache.carbondata.core.datamap.{DataMapRegistry, DataMapStoreManager}
import org.apache.carbondata.core.datamap.status.DataMapStatusManager
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.datamap.{DataMapManager, IndexDataMapRebuildRDD}
+import org.apache.carbondata.events.{UpdateDataMapPostExecutionEvent, _}
/**
* Rebuild the datamaps through sync with main table data. After sync with parent table's it enables
@@ -49,7 +51,17 @@ case class CarbonDataMapRebuildCommand(
provider.rebuild()
// After rebuild successfully enable the datamap.
+ val operationContext: OperationContext = new OperationContext()
+ val systemFolderLocation: String = CarbonProperties.getInstance().getSystemFolderLocation
+ val updateDataMapPreExecutionEvent: UpdateDataMapPreExecutionEvent =
+ new UpdateDataMapPreExecutionEvent(sparkSession, systemFolderLocation)
+ OperationListenerBus.getInstance().fireEvent(updateDataMapPreExecutionEvent,
+ operationContext)
DataMapStatusManager.enableDataMap(dataMapName)
+ val updateDataMapPostExecutionEvent: UpdateDataMapPostExecutionEvent =
+ new UpdateDataMapPostExecutionEvent(sparkSession, systemFolderLocation)
+ OperationListenerBus.getInstance().fireEvent(updateDataMapPostExecutionEvent,
+ operationContext)
Seq.empty
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b3f78206/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index f1ed5d1..722119e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
@@ -35,6 +35,7 @@ import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema}
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.datamap.{DataMapManager, IndexDataMapProvider}
import org.apache.carbondata.events._
@@ -197,7 +198,17 @@ case class CarbonDropDataMapCommand(
if (dataMapSchema != null) {
dataMapProvider =
DataMapManager.get.getDataMapProvider(mainTable, dataMapSchema, sparkSession)
+ val operationContext: OperationContext = new OperationContext()
+ val systemFolderLocation: String = CarbonProperties.getInstance().getSystemFolderLocation
+ val updateDataMapPreExecutionEvent: UpdateDataMapPreExecutionEvent =
+ UpdateDataMapPreExecutionEvent(sparkSession, systemFolderLocation)
+ OperationListenerBus.getInstance().fireEvent(updateDataMapPreExecutionEvent,
+ operationContext)
DataMapStatusManager.dropDataMap(dataMapSchema.getDataMapName)
+ val updateDataMapPostExecutionEvent: UpdateDataMapPostExecutionEvent =
+ UpdateDataMapPostExecutionEvent(sparkSession, systemFolderLocation)
+ OperationListenerBus.getInstance().fireEvent(updateDataMapPostExecutionEvent,
+ operationContext)
// if it is indexDataMap provider like lucene, then call cleanData, which will launch a job
// to clear datamap from memory(clears from segmentMap and cache), This is called before
// deleting the datamap schemas from _System folder
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b3f78206/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 69db3ea..38bdbcf 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -62,7 +62,7 @@ import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, TupleIdEnum}
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil, DataTypeUtil, ObjectSerializationUtil}
import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.events.{OperationContext, OperationListenerBus}
+import org.apache.carbondata.events.{BuildDataMapPostExecutionEvent, BuildDataMapPreExecutionEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.events.exception.PreEventException
import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.TableProcessingOperations
@@ -233,6 +233,18 @@ case class CarbonLoadDataCommand(
isOverwriteTable)
operationContext.setProperty("isOverwrite", isOverwriteTable)
OperationListenerBus.getInstance.fireEvent(loadTablePreExecutionEvent, operationContext)
+ // Add pre event listener for index datamap
+ val tableDataMaps = DataMapStoreManager.getInstance().getAllDataMap(table)
+ val dataMapOperationContext = new OperationContext()
+ if (null != tableDataMaps) {
+ val dataMapNames: mutable.Buffer[String] =
+ tableDataMaps.asScala.map(dataMap => dataMap.getDataMapSchema.getDataMapName)
+ val buildDataMapPreExecutionEvent: BuildDataMapPreExecutionEvent =
+ new BuildDataMapPreExecutionEvent(sparkSession,
+ table.getAbsoluteTableIdentifier, dataMapNames)
+ OperationListenerBus.getInstance().fireEvent(buildDataMapPreExecutionEvent,
+ dataMapOperationContext)
+ }
// First system has to partition the data first and then call the load data
LOGGER.info(s"Initiating Direct Load for the Table : ($dbName.$tableName)")
// Clean up the old invalid segment data before creating a new entry for new load.
@@ -300,6 +312,13 @@ case class CarbonLoadDataCommand(
table.getCarbonTableIdentifier,
carbonLoadModel)
OperationListenerBus.getInstance.fireEvent(loadTablePostExecutionEvent, operationContext)
+ if (null != tableDataMaps) {
+ val buildDataMapPostExecutionEvent: BuildDataMapPostExecutionEvent =
+ BuildDataMapPostExecutionEvent(sparkSession, table.getAbsoluteTableIdentifier)
+ OperationListenerBus.getInstance()
+ .fireEvent(buildDataMapPostExecutionEvent, dataMapOperationContext)
+ }
+
} catch {
case CausedBy(ex: NoRetryException) =>
// update the load entry in table status file for changing the status to marked for delete
[23/50] [abbrv] carbondata git commit: [HOTFIX] fix java style errors
Posted by ja...@apache.org.
[HOTFIX] fix java style errors
This closes #2371
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ff036459
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ff036459
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ff036459
Branch: refs/heads/carbonstore
Commit: ff0364599c5bc1d403b4294237e6c88a254beaf4
Parents: 685087e
Author: Zhang Zhichao <44...@qq.com>
Authored: Wed Jun 13 12:13:41 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jun 13 15:54:54 2018 +0800
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +-
.../command/management/CarbonLoadDataCommand.scala | 2 ++
.../processing/loading/model/CarbonLoadModel.java | 2 +-
.../processing/loading/model/CarbonLoadModelBuilder.java | 3 ++-
.../carbondata/processing/loading/model/LoadOption.java | 5 +++--
.../carbondata/processing/util/CarbonLoaderUtil.java | 10 +++++-----
6 files changed, 14 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ff036459/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 1f04fa4..61a5b42 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -885,7 +885,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
"ALL_DICTIONARY_PATH", "MAXCOLUMNS", "COMMENTCHAR", "DATEFORMAT", "BAD_RECORD_PATH",
"BATCH_SORT_SIZE_INMB", "GLOBAL_SORT_PARTITIONS", "SINGLE_PASS",
"IS_EMPTY_DATA_BAD_RECORD", "HEADER", "TIMESTAMPFORMAT", "SKIP_EMPTY_LINE",
- "SORT_COLUMN_BOUNDS","LOAD_MIN_SIZE_INMB"
+ "SORT_COLUMN_BOUNDS", "LOAD_MIN_SIZE_INMB"
)
var isSupported = true
val invalidOptions = StringBuilder.newBuilder
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ff036459/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 4703b23..69db3ea 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -24,6 +24,7 @@ import java.util.UUID
import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
+
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.conf.Configuration
import org.apache.spark.rdd.RDD
@@ -44,6 +45,7 @@ import org.apache.spark.sql.types._
import org.apache.spark.storage.StorageLevel
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.{CarbonReflectionUtils, CausedBy, FileUtils}
+
import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ff036459/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index f267fa7..90c297e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -210,7 +210,7 @@ public class CarbonLoadModel implements Serializable {
* Flder path to where data should be written for this load.
*/
private String dataWritePath;
-
+
/**
* sort columns bounds
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ff036459/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
index 4ad1984..a88ce60 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
@@ -270,7 +270,8 @@ public class CarbonLoadModelBuilder {
carbonLoadModel.setMaxColumns(String.valueOf(validatedMaxColumns));
carbonLoadModel.readAndSetLoadMetadataDetails();
carbonLoadModel.setSortColumnsBoundsStr(optionsFinal.get("sort_column_bounds"));
- carbonLoadModel.setLoadMinSize(optionsFinal.get(CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB));
+ carbonLoadModel.setLoadMinSize(
+ optionsFinal.get(CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB));
}
private int validateMaxColumns(String[] csvHeaders, String maxColumns)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ff036459/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
index 1a65937..17c3651 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
@@ -196,8 +196,9 @@ public class LoadOption {
optionsFinal.put("single_pass", String.valueOf(singlePass));
optionsFinal.put("sort_scope", "local_sort");
optionsFinal.put("sort_column_bounds", Maps.getOrDefault(options, "sort_column_bounds", ""));
- optionsFinal.put(CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB, Maps.getOrDefault(options,CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB, CarbonCommonConstants
- .CARBON_LOAD_MIN_NODE_SIZE_INMB_DEFAULT));
+ optionsFinal.put(CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB,
+ Maps.getOrDefault(options,CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB,
+ CarbonCommonConstants.CARBON_LOAD_MIN_NODE_SIZE_INMB_DEFAULT));
return optionsFinal;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ff036459/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index d5a0b78..0ea7223 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -582,7 +582,7 @@ public final class CarbonLoaderUtil {
*/
public static Map<String, List<Distributable>> nodeBlockMapping(
List<Distributable> blockInfos, int noOfNodesInput, List<String> activeNodes,
- BlockAssignmentStrategy blockAssignmentStrategy, String expectedMinSizePerNode ) {
+ BlockAssignmentStrategy blockAssignmentStrategy, String expectedMinSizePerNode) {
ArrayList<NodeMultiBlockRelation> rtnNode2Blocks = new ArrayList<>();
Set<Distributable> uniqueBlocks = new HashSet<>(blockInfos);
@@ -630,13 +630,13 @@ public final class CarbonLoaderUtil {
} else {
blockAssignmentStrategy = BlockAssignmentStrategy.BLOCK_NUM_FIRST;
}
- LOGGER.info("Specified minimum data size to load is less than the average size for each node, "
- + "fallback to default strategy" + blockAssignmentStrategy);
+ LOGGER.info("Specified minimum data size to load is less than the average size "
+ + "for each node, fallback to default strategy" + blockAssignmentStrategy);
} else {
sizePerNode = iexpectedMinSizePerNode;
}
}
-
+
if (BlockAssignmentStrategy.NODE_MIN_SIZE_FIRST == blockAssignmentStrategy) {
// assign blocks to each node ignore data locality
assignBlocksIgnoreDataLocality(rtnNode2Blocks, sizePerNode, uniqueBlocks, activeNodes);
@@ -658,7 +658,7 @@ public final class CarbonLoaderUtil {
}
return rtnNodeBlocksMap;
}
-
+
/**
* Assigning the blocks of a node to tasks.
*
[21/50] [abbrv] carbondata git commit: [CARBONDATA-2603] Fix: error
handling during reader build failure
Posted by ja...@apache.org.
[CARBONDATA-2603] Fix: error handling during reader build failure
problem :
When the CarbonReaderBuilder.build() is failed due to some problems like invalid
projection that leads to query model creation failure. Blocklet datamap is not cleared for that table.So,
the next reader instance uses old blocklet datamap . That creates error.
Solution: Clear the blocklet datamap if the reader build is failed.
This closes #2368
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9b88a065
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9b88a065
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9b88a065
Branch: refs/heads/carbonstore
Commit: 9b88a065276c97e17ae6fa1c6f7a27b0d6d12a7c
Parents: 19312ab
Author: ajantha-bhat <aj...@gmail.com>
Authored: Mon Jun 11 19:17:33 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Tue Jun 12 20:12:09 2018 +0530
----------------------------------------------------------------------
.../sdk/file/CarbonReaderBuilder.java | 39 ++++++++-----
.../carbondata/sdk/file/CarbonReaderTest.java | 61 ++++++++++++++++++++
2 files changed, 84 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b88a065/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
index 83cb34e..ebee41a 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
@@ -24,6 +24,7 @@ import java.util.Objects;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.datamap.DataMapStoreManager;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.scan.expression.Expression;
@@ -200,23 +201,29 @@ public class CarbonReaderBuilder {
format.setColumnProjection(job.getConfiguration(), projectionColumns);
}
- final List<InputSplit> splits =
- format.getSplits(new JobContextImpl(job.getConfiguration(), new JobID()));
-
- List<RecordReader<Void, T>> readers = new ArrayList<>(splits.size());
- for (InputSplit split : splits) {
- TaskAttemptContextImpl attempt =
- new TaskAttemptContextImpl(job.getConfiguration(), new TaskAttemptID());
- RecordReader reader = format.createRecordReader(split, attempt);
- try {
- reader.initialize(split, attempt);
- readers.add(reader);
- } catch (Exception e) {
- reader.close();
- throw e;
+ try {
+ final List<InputSplit> splits =
+ format.getSplits(new JobContextImpl(job.getConfiguration(), new JobID()));
+
+ List<RecordReader<Void, T>> readers = new ArrayList<>(splits.size());
+ for (InputSplit split : splits) {
+ TaskAttemptContextImpl attempt =
+ new TaskAttemptContextImpl(job.getConfiguration(), new TaskAttemptID());
+ RecordReader reader = format.createRecordReader(split, attempt);
+ try {
+ reader.initialize(split, attempt);
+ readers.add(reader);
+ } catch (Exception e) {
+ reader.close();
+ throw e;
+ }
}
+ return new CarbonReader<>(readers);
+ } catch (Exception ex) {
+ // Clear the datamap cache as it can get added in getSplits() method
+ DataMapStoreManager.getInstance()
+ .clearDataMaps(table.getAbsoluteTableIdentifier());
+ throw ex;
}
-
- return new CarbonReader<>(readers);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b88a065/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index fb2e2bc..2bc4b1f 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -353,6 +353,67 @@ public class CarbonReaderTest extends TestCase {
}
@Test
+ public void testWriteAndReadFilesWithReaderBuildFail() throws IOException, InterruptedException {
+ String path1 = "./testWriteFiles";
+ String path2 = "./testWriteFiles2";
+ FileUtils.deleteDirectory(new File(path1));
+ FileUtils.deleteDirectory(new File(path2));
+
+ Field[] fields = new Field[] { new Field("c1", "string"),
+ new Field("c2", "int") };
+ Schema schema = new Schema(fields);
+ CarbonWriterBuilder builder = CarbonWriter.builder();
+
+ CarbonWriter carbonWriter = null;
+ try {
+ carbonWriter = builder.outputPath(path1).isTransactionalTable(false).uniqueIdentifier(12345)
+ .buildWriterForCSVInput(schema);
+ } catch (InvalidLoadOptionException e) {
+ e.printStackTrace();
+ }
+ carbonWriter.write(new String[] { "MNO", "100" });
+ carbonWriter.close();
+
+ Field[] fields1 = new Field[] { new Field("p1", "string"),
+ new Field("p2", "int") };
+ Schema schema1 = new Schema(fields1);
+ CarbonWriterBuilder builder1 = CarbonWriter.builder();
+
+ CarbonWriter carbonWriter1 = null;
+ try {
+ carbonWriter1 = builder1.outputPath(path2).isTransactionalTable(false).uniqueIdentifier(12345)
+ .buildWriterForCSVInput(schema1);
+ } catch (InvalidLoadOptionException e) {
+ e.printStackTrace();
+ }
+ carbonWriter1.write(new String[] { "PQR", "200" });
+ carbonWriter1.close();
+
+ try {
+ CarbonReader reader =
+ CarbonReader.builder(path1, "_temp").
+ projection(new String[] { "c1", "c3" })
+ .isTransactionalTable(false).build();
+ } catch (Exception e){
+ System.out.println("Success");
+ }
+ CarbonReader reader1 =
+ CarbonReader.builder(path2, "_temp1")
+ .projection(new String[] { "p1", "p2" })
+ .isTransactionalTable(false).build();
+
+ while (reader1.hasNext()) {
+ Object[] row1 = (Object[]) reader1.readNextRow();
+ System.out.println(row1[0]);
+ System.out.println(row1[1]);
+ }
+ reader1.close();
+
+ FileUtils.deleteDirectory(new File(path1));
+ FileUtils.deleteDirectory(new File(path2));
+ }
+
+ @Test
public void testReadColumnTwice() throws IOException, InterruptedException {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
[32/50] [abbrv] carbondata git commit: [CARBONDATA-2553] support ZSTD
compression for sort temp file
Posted by ja...@apache.org.
[CARBONDATA-2553] support ZSTD compression for sort temp file
This closes #2350
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ece06729
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ece06729
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ece06729
Branch: refs/heads/carbonstore
Commit: ece0672930b8bffba8e9bddad63560ff9d6cd582
Parents: 5593d16
Author: Manhua <ke...@qq.com>
Authored: Tue May 29 09:21:52 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Mon Jun 18 21:31:02 2018 +0800
----------------------------------------------------------------------
core/pom.xml | 5 ++
.../core/constants/CarbonCommonConstants.java | 2 +-
.../datastore/filesystem/LocalCarbonFile.java | 8 +++
.../carbondata/core/util/CarbonProperties.java | 4 +-
docs/useful-tips-on-carbondata.md | 2 +-
.../TestLoadWithSortTempCompressed.scala | 51 +++++++++++++++++++-
6 files changed, 66 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ece06729/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 7d87037..c145c3b 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -68,6 +68,11 @@
<version>${snappy.version}</version>
</dependency>
<dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ <version>1.3.2-2</version>
+ </dependency>
+ <dependency>
<groupId>org.jmockit</groupId>
<artifactId>jmockit</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ece06729/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 2fcf0f5..355bcb6 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1378,7 +1378,7 @@ public final class CarbonCommonConstants {
public static final String CARBON_SORT_TEMP_COMPRESSOR = "carbon.sort.temp.compressor";
/**
- * The optional values are 'SNAPPY','GZIP','BZIP2','LZ4'.
+ * The optional values are 'SNAPPY','GZIP','BZIP2','LZ4','ZSTD'.
* By default, empty means that Carbondata will not compress the sort temp files.
*/
public static final String CARBON_SORT_TEMP_COMPRESSOR_DEFAULT = "";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ece06729/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
index 60b7e17..5b6f657 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
@@ -42,6 +42,8 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
+import com.github.luben.zstd.ZstdInputStream;
+import com.github.luben.zstd.ZstdOutputStream;
import net.jpountz.lz4.LZ4BlockInputStream;
import net.jpountz.lz4.LZ4BlockOutputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
@@ -290,6 +292,8 @@ public class LocalCarbonFile implements CarbonFile {
inputStream = new SnappyInputStream(new FileInputStream(path));
} else if ("LZ4".equalsIgnoreCase(compressor)) {
inputStream = new LZ4BlockInputStream(new FileInputStream(path));
+ } else if ("ZSTD".equalsIgnoreCase(compressor)) {
+ inputStream = new ZstdInputStream(new FileInputStream(path));
} else {
throw new IOException("Unsupported compressor: " + compressor);
}
@@ -368,6 +372,10 @@ public class LocalCarbonFile implements CarbonFile {
outputStream = new SnappyOutputStream(new FileOutputStream(path));
} else if ("LZ4".equalsIgnoreCase(compressor)) {
outputStream = new LZ4BlockOutputStream(new FileOutputStream(path));
+ } else if ("ZSTD".equalsIgnoreCase(compressor)) {
+ // compression level 1 is cost-effective for sort temp file
+ // which is not used for storage
+ outputStream = new ZstdOutputStream(new FileOutputStream(path), 1);
} else {
throw new IOException("Unsupported compressor: " + compressor);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ece06729/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index b134a7c..dc50ab0 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -1290,11 +1290,11 @@ public final class CarbonProperties {
String compressor = getProperty(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR,
CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR_DEFAULT).toUpperCase();
if (compressor.isEmpty() || "SNAPPY".equals(compressor) || "GZIP".equals(compressor)
- || "BZIP2".equals(compressor) || "LZ4".equals(compressor)) {
+ || "BZIP2".equals(compressor) || "LZ4".equals(compressor) || "ZSTD".equals(compressor)) {
return compressor;
} else {
LOGGER.warn("The ".concat(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR)
- .concat(" configuration value is invalid. Only snappy,gzip,bip2,lz4 and")
+ .concat(" configuration value is invalid. Only snappy, gzip, bip2, lz4, zstd and")
.concat(" empty are allowed. It will not compress the sort temp files by default"));
return CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR_DEFAULT;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ece06729/docs/useful-tips-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/useful-tips-on-carbondata.md b/docs/useful-tips-on-carbondata.md
index 732d38f..d00f785 100644
--- a/docs/useful-tips-on-carbondata.md
+++ b/docs/useful-tips-on-carbondata.md
@@ -168,7 +168,7 @@
| carbon.detail.batch.size | spark/carbonlib/carbon.properties | Data loading | The buffer size to store records, returned from the block scan. | In limit scenario this parameter is very important. For example your query limit is 1000. But if we set this value to 3000 that means we get 3000 records from scan but spark will only take 1000 rows. So the 2000 remaining are useless. In one Finance test case after we set it to 100, in the limit 1000 scenario the performance increase about 2 times in comparison to if we set this value to 12000. |
| carbon.use.local.dir | spark/carbonlib/carbon.properties | Data loading | Whether use YARN local directories for multi-table load disk load balance | If this is set it to true CarbonData will use YARN local directories for multi-table load disk load balance, that will improve the data load performance. |
| carbon.use.multiple.temp.dir | spark/carbonlib/carbon.properties | Data loading | Whether to use multiple YARN local directories during table data loading for disk load balance | After enabling 'carbon.use.local.dir', if this is set to true, CarbonData will use all YARN local directories during data load for disk load balance, that will improve the data load performance. Please enable this property when you encounter disk hotspot problem during data loading. |
- | carbon.sort.temp.compressor | spark/carbonlib/carbon.properties | Data loading | Specify the name of compressor to compress the intermediate sort temporary files during sort procedure in data loading. | The optional values are 'SNAPPY','GZIP','BZIP2','LZ4' and empty. By default, empty means that Carbondata will not compress the sort temp files. This parameter will be useful if you encounter disk bottleneck. |
+ | carbon.sort.temp.compressor | spark/carbonlib/carbon.properties | Data loading | Specify the name of compressor to compress the intermediate sort temporary files during sort procedure in data loading. | The optional values are 'SNAPPY','GZIP','BZIP2','LZ4','ZSTD' and empty. By default, empty means that Carbondata will not compress the sort temp files. This parameter will be useful if you encounter disk bottleneck. |
| carbon.load.skewedDataOptimization.enabled | spark/carbonlib/carbon.properties | Data loading | Whether to enable size based block allocation strategy for data loading. | When loading, carbondata will use file size based block allocation strategy for task distribution. It will make sure that all the executors process the same size of data -- It's useful if the size of your input data files varies widely, say 1MB~1GB. |
| carbon.load.min.size.enabled | spark/carbonlib/carbon.properties | Data loading | Whether to enable node minumun input data size allocation strategy for data loading.| When loading, carbondata will use node minumun input data size allocation strategy for task distribution. It will make sure the node load the minimum amount of data -- It's useful if the size of your input data files very small, say 1MB~256MB,Avoid generating a large number of small files. |
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ece06729/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadWithSortTempCompressed.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadWithSortTempCompressed.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadWithSortTempCompressed.scala
index 61acea4..5fbdd14 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadWithSortTempCompressed.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadWithSortTempCompressed.scala
@@ -50,9 +50,8 @@ class TestLoadWithSortTempCompressed extends QueryTest
override protected def beforeAll(): Unit = {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR,
- "SNAPPY")
}
+
override def afterAll(): Unit = {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR,
@@ -84,6 +83,8 @@ class TestLoadWithSortTempCompressed extends QueryTest
test("test data load for simple table with sort temp compressed with snappy" +
" and off-heap sort enabled") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR,
+ "SNAPPY")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
testSimpleTable()
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
@@ -92,6 +93,28 @@ class TestLoadWithSortTempCompressed extends QueryTest
test("test data load for simple table with sort temp compressed with snappy" +
" and off-heap sort disabled") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR,
+ "SNAPPY")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "false")
+ testSimpleTable()
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
+ originOffHeapStatus)
+ }
+
+ test("test data load for simple table with sort temp compressed with zstd" +
+ " and off-heap sort enabled") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR,
+ "ZSTD")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
+ testSimpleTable()
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
+ originOffHeapStatus)
+ }
+
+ test("test data load for simple table with sort temp compressed with zstd" +
+ " and off-heap sort disabled") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR,
+ "ZSTD")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "false")
testSimpleTable()
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
@@ -138,6 +161,8 @@ class TestLoadWithSortTempCompressed extends QueryTest
test("test data load for complex table with sort temp compressed with snappy" +
" and off-heap sort enabled") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR,
+ "SNAPPY")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
testComplexTable()
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
@@ -146,6 +171,28 @@ class TestLoadWithSortTempCompressed extends QueryTest
test("test data load for complex table with sort temp compressed with snappy" +
" and off-heap sort disabled") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR,
+ "SNAPPY")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "false")
+ testComplexTable()
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
+ originOffHeapStatus)
+ }
+
+ test("test data load for complex table with sort temp compressed with zstd" +
+ " and off-heap sort enabled") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR,
+ "ZSTD")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
+ testComplexTable()
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
+ originOffHeapStatus)
+ }
+
+ test("test data load for complex table with sort temp compressed with zstd" +
+ " and off-heap sort disabled") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR,
+ "ZSTD")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "false")
testComplexTable()
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
[42/50] [abbrv] carbondata git commit: [CARBONDATA-2617] Invalid
tuple-id and block id getting formed for Non partition table
Posted by ja...@apache.org.
[CARBONDATA-2617] Invalid tuple-id and block id getting formed for Non partition table
Problem
Invalid tuple and block id getting formed for non partition table
Analysis
While creating a partition table a segment file was written in the Metadata folder under table structure. This was introduced during
development of partition table feature. At that time segment file was written only for partition table and it was used to distinguish between
parition and non partition table in the code. But later the code was modified to write the segment file for both parititon and non partition
table and the code to distinguish partition and non partition table was not modified which is causing this incorrect formation of block and tuple id.
Fix
Modify the logic to distinguish partitioned and non partitioned table and the same has been handled in this PR.
This closes #2385
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0e1d550e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0e1d550e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0e1d550e
Branch: refs/heads/carbonstore
Commit: 0e1d550e8dacba798e9ffbdda25c4388e8933632
Parents: dc53dee
Author: rahul <ra...@knoldus.in>
Authored: Tue Jun 19 19:23:26 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Wed Jun 20 16:37:23 2018 +0530
----------------------------------------------------------------------
.../core/mutate/CarbonUpdateUtil.java | 4 +-
.../executor/impl/AbstractQueryExecutor.java | 4 +-
.../SegmentUpdateStatusManager.java | 20 ++---
.../apache/carbondata/core/util/CarbonUtil.java | 4 +-
.../iud/DeleteCarbonTableTestCase.scala | 83 ++++++++++++++++++++
.../command/mutation/DeleteExecution.scala | 6 +-
6 files changed, 100 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e1d550e/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index 40d498c..8627bdb 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -81,10 +81,10 @@ public class CarbonUpdateUtil {
/**
* Returns block path from tuple id
*/
- public static String getTableBlockPath(String tid, String tablePath, boolean isSegmentFile) {
+ public static String getTableBlockPath(String tid, String tablePath, boolean isPartitionTable) {
String partField = getRequiredFieldFromTID(tid, TupleIdEnum.PART_ID);
// If it has segment file then partfield can be appended directly to table path
- if (isSegmentFile) {
+ if (isPartitionTable) {
return tablePath + CarbonCommonConstants.FILE_SEPARATOR + partField.replace("#", "/");
}
String part = CarbonTablePath.addPartPrefix(partField);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e1d550e/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 2bbe75c..f365045 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -296,8 +296,8 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
String blockId = CarbonUtil
.getBlockId(queryModel.getAbsoluteTableIdentifier(), filePath, segment.getSegmentNo(),
queryModel.getTable().getTableInfo().isTransactionalTable(),
- segment.getSegmentFileName() != null);
- if (segment.getSegmentFileName() != null) {
+ queryModel.getTable().isHivePartitionTable());
+ if (queryModel.getTable().isHivePartitionTable()) {
blockExecutionInfo.setBlockId(CarbonTablePath.getShortBlockIdForPartitionTable(blockId));
} else {
blockExecutionInfo.setBlockId(CarbonTablePath.getShortBlockId(blockId));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e1d550e/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index eb850e4..1b43b65 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -69,6 +69,7 @@ public class SegmentUpdateStatusManager {
private LoadMetadataDetails[] segmentDetails;
private SegmentUpdateDetails[] updateDetails;
private Map<String, SegmentUpdateDetails> blockAndDetailsMap;
+ private boolean isPartitionTable;
public SegmentUpdateStatusManager(CarbonTable table,
LoadMetadataDetails[] segmentDetails) {
@@ -77,6 +78,7 @@ public class SegmentUpdateStatusManager {
// on latest file status.
this.segmentDetails = segmentDetails;
updateDetails = readLoadMetadata();
+ isPartitionTable = table.isHivePartitionTable();
populateMap();
}
@@ -96,6 +98,7 @@ public class SegmentUpdateStatusManager {
} else {
updateDetails = new SegmentUpdateDetails[0];
}
+ isPartitionTable = table.isHivePartitionTable();
populateMap();
}
@@ -246,29 +249,22 @@ public class SegmentUpdateStatusManager {
* @throws Exception
*/
public String[] getDeleteDeltaFilePath(String blockFilePath, String segmentId) throws Exception {
- String segmentFile = null;
- for (LoadMetadataDetails segmentDetail : segmentDetails) {
- if (segmentDetail.getLoadName().equals(segmentId)) {
- segmentFile = segmentDetail.getSegmentFile();
- break;
- }
- }
String blockId =
- CarbonUtil.getBlockId(identifier, blockFilePath, segmentId, true, segmentFile != null);
+ CarbonUtil.getBlockId(identifier, blockFilePath, segmentId, true, isPartitionTable);
String tupleId;
- if (segmentFile != null) {
+ if (isPartitionTable) {
tupleId = CarbonTablePath.getShortBlockIdForPartitionTable(blockId);
} else {
tupleId = CarbonTablePath.getShortBlockId(blockId);
}
- return getDeltaFiles(tupleId, CarbonCommonConstants.DELETE_DELTA_FILE_EXT, segmentFile)
+ return getDeltaFiles(tupleId, CarbonCommonConstants.DELETE_DELTA_FILE_EXT)
.toArray(new String[0]);
}
/**
* Returns all delta file paths of specified block
*/
- private List<String> getDeltaFiles(String tupleId, String extension, String segmentFile)
+ private List<String> getDeltaFiles(String tupleId, String extension)
throws Exception {
String segment = CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.SEGMENT_ID);
String completeBlockName = CarbonTablePath.addDataPartPrefix(
@@ -276,7 +272,7 @@ public class SegmentUpdateStatusManager {
+ CarbonCommonConstants.FACT_FILE_EXT);
String blockPath;
- if (segmentFile != null) {
+ if (isPartitionTable) {
blockPath = identifier.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR
+ CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.PART_ID)
.replace("#", "/") + CarbonCommonConstants.FILE_SEPARATOR + completeBlockName;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e1d550e/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 1f6c697..4e2c16f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2977,13 +2977,13 @@ public final class CarbonUtil {
* @return
*/
public static String getBlockId(AbsoluteTableIdentifier identifier, String filePath,
- String segmentId, boolean isTransactionalTable, boolean hasSegmentFile) {
+ String segmentId, boolean isTransactionalTable, boolean isPartitionTable) {
String blockId;
String blockName = filePath.substring(filePath.lastIndexOf("/") + 1, filePath.length());
String tablePath = identifier.getTablePath();
if (filePath.startsWith(tablePath)) {
- if (!isTransactionalTable || !hasSegmentFile) {
+ if (!isTransactionalTable || !isPartitionTable) {
blockId = "Part0" + CarbonCommonConstants.FILE_SEPARATOR + "Segment_" + segmentId
+ CarbonCommonConstants.FILE_SEPARATOR + blockName;
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e1d550e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
index 1fbddb0..64aae1d 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
@@ -18,12 +18,16 @@ package org.apache.carbondata.spark.testsuite.iud
import java.io.File
+import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.test.Spark2TestQueryExecutor
import org.apache.spark.sql.test.util.QueryTest
import org.apache.spark.sql.{CarbonEnv, Row, SaveMode}
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil
+import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -120,6 +124,7 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
Seq(Row(2), Row(3),Row(4), Row(5))
)
}
+
test("partition delete data from carbon table[where clause ]") {
sql("""drop table if exists iud_db.dest""")
sql("""create table iud_db.dest (c1 string,c2 int,c5 string) PARTITIONED BY(c3 string) STORED BY 'org.apache.carbondata.format'""")
@@ -214,6 +219,84 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("drop table update_status_files")
}
+ test("tuple-id for partition table ") {
+ sql("drop table if exists iud_db.dest_tuple_part")
+ sql(
+ """create table iud_db.dest_tuple_part (c1 string,c2 int,c5 string) PARTITIONED BY(c3 string) STORED BY 'org.apache.carbondata.format'""".stripMargin)
+ sql(
+ s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud_db.dest_tuple_part""".stripMargin)
+ sql("drop table if exists iud_db.dest_tuple")
+ sql(
+ """create table iud_db.dest_tuple (c1 string,c2 int,c5 string,c3 string) STORED BY 'org.apache.carbondata.format'""".stripMargin)
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud_db.dest_tuple""")
+
+ val dataframe_part = sql("select getTupleId() as tupleId from iud_db.dest_tuple_part").collect()
+ val listOfTupleId_part = dataframe_part.map(df => df.get(0).toString).sorted
+ assert(listOfTupleId_part(0).startsWith("c3=aa/0/0-100100000100001_batchno0-0-0-") &&
+ listOfTupleId_part(0).endsWith("/0/0/0"))
+ assert(listOfTupleId_part(1).startsWith("c3=bb/0/0-100100000100002_batchno0-0-0-") &&
+ listOfTupleId_part(1).endsWith("/0/0/0"))
+ assert(listOfTupleId_part(2).startsWith("c3=cc/0/0-100100000100003_batchno0-0-0-") &&
+ listOfTupleId_part(2).endsWith("/0/0/0"))
+ assert(listOfTupleId_part(3).startsWith("c3=dd/0/0-100100000100004_batchno0-0-0-") &&
+ listOfTupleId_part(3).endsWith("/0/0/0"))
+ assert(listOfTupleId_part(4).startsWith("c3=ee/0/0-100100000100005_batchno0-0-0-") &&
+ listOfTupleId_part(4).endsWith("/0/0/0"))
+
+ val dataframe = sql("select getTupleId() as tupleId from iud_db.dest_tuple")
+ val listOfTupleId = dataframe.collect().map(df => df.get(0).toString).sorted
+ assert(
+ listOfTupleId(0).startsWith("0/0/0-0_batchno0-0-0-") && listOfTupleId(0).endsWith("/0/0/0"))
+ assert(
+ listOfTupleId(1).startsWith("0/0/0-0_batchno0-0-0-") && listOfTupleId(1).endsWith("/0/0/1"))
+ assert(
+ listOfTupleId(2).startsWith("0/0/0-0_batchno0-0-0-") && listOfTupleId(2).endsWith("/0/0/2"))
+ assert(
+ listOfTupleId(3).startsWith("0/0/0-0_batchno0-0-0-") && listOfTupleId(3).endsWith("/0/0/3"))
+ assert(
+ listOfTupleId(4).startsWith("0/0/0-0_batchno0-0-0-") && listOfTupleId(4).endsWith("/0/0/4"))
+
+ val carbonTable_part = CarbonEnv.getInstance(Spark2TestQueryExecutor.spark).carbonMetastore
+ .lookupRelation(Option("iud_db"), "dest_tuple_part")(Spark2TestQueryExecutor.spark)
+ .asInstanceOf[CarbonRelation].carbonTable
+
+ val carbonTable = CarbonEnv.getInstance(Spark2TestQueryExecutor.spark).carbonMetastore
+ .lookupRelation(Option("iud_db"), "dest_tuple")(Spark2TestQueryExecutor.spark)
+ .asInstanceOf[CarbonRelation].carbonTable
+
+ val carbonDataFilename = new File(carbonTable.getTablePath + "/Fact/Part0/Segment_0/")
+ .listFiles().filter(fn => fn.getName.endsWith(".carbondata"))
+ val blockId = CarbonUtil.getBlockId(carbonTable.getAbsoluteTableIdentifier,
+ carbonDataFilename(0).getAbsolutePath,
+ "0",
+ carbonTable.isTransactionalTable,
+ carbonTable.isHivePartitionTable)
+
+ assert(blockId.startsWith("Part0/Segment_0/part-0-0_batchno0-0-0-"))
+ val carbonDataFilename_part = new File(carbonTable_part.getTablePath + "/c3=aa").listFiles()
+ .filter(fn => fn.getName.endsWith(".carbondata"))
+ val blockId_part = CarbonUtil.getBlockId(carbonTable.getAbsoluteTableIdentifier,
+ carbonDataFilename_part(0).getAbsolutePath,
+ "0",
+ carbonTable.isTransactionalTable,
+ carbonTable.isHivePartitionTable)
+ assert(blockId_part.startsWith("Part0/Segment_0/part-0-100100000100001_batchno0-0-0-"))
+
+ val tableBlockPath = CarbonUpdateUtil
+ .getTableBlockPath(listOfTupleId(0),
+ carbonTable.getTablePath,
+ carbonTable.isHivePartitionTable)
+ val tableBl0ckPath_part = CarbonUpdateUtil
+ .getTableBlockPath(listOfTupleId_part(0),
+ carbonTable_part.getTablePath,
+ carbonTable_part.isHivePartitionTable)
+ assert(tableBl0ckPath_part.endsWith("iud_db.db/dest_tuple_part/c3=aa"))
+ assert(tableBlockPath.endsWith("iud_db.db/dest_tuple/Fact/Part0/Segment_0"))
+
+ sql("drop table if exists iud_db.dest_tuple_part")
+ sql("drop table if exists iud_db.dest_tuple")
+
+ }
override def afterAll {
sql("use default")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e1d550e/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
index 127e1b1..df3b961 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
@@ -134,7 +134,7 @@ object DeleteExecution {
groupedRows.toIterator,
timestamp,
rowCountDetailsVO,
- segmentFile)
+ carbonTable.isHivePartitionTable)
}
result
}
@@ -222,7 +222,7 @@ object DeleteExecution {
iter: Iterator[Row],
timestamp: String,
rowCountDetailsVO: RowCountDetailsVO,
- segmentFile: String
+ isPartitionTable: Boolean
): Iterator[(SegmentStatus, (SegmentUpdateDetails, ExecutionErrors))] = {
val result = new DeleteDelataResultImpl()
@@ -258,7 +258,7 @@ object DeleteExecution {
countOfRows = countOfRows + 1
}
- val blockPath = CarbonUpdateUtil.getTableBlockPath(TID, tablePath, segmentFile != null)
+ val blockPath = CarbonUpdateUtil.getTableBlockPath(TID, tablePath, isPartitionTable)
val completeBlockName = CarbonTablePath
.addDataPartPrefix(CarbonUpdateUtil.getRequiredFieldFromTID(TID, TupleIdEnum.BLOCK_ID) +
CarbonCommonConstants.FACT_FILE_EXT)
[24/50] [abbrv] carbondata git commit: [CARBONDATA-2604] Getting
ArrayIndexOutOfBoundException during compaction after IUD in cluster is fixed
Posted by ja...@apache.org.
[CARBONDATA-2604] Getting ArrayIndexOutOfBoundException during compaction after IUD in cluster is fixed
Issue: if some records are deleted then during filling the measure and dimension data no of valid rows count and actual rows may be different if
some records are deleted . and during filling of measure data it will iterrate over the scanned result. so it will cause ArrayIndexOutOfBoundException
Solution : Make a new temp list to collect the measure and dimension data during scan and fill inside RawBasedResultCollector and add it to final list
This closes #2369
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/efad40d5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/efad40d5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/efad40d5
Branch: refs/heads/carbonstore
Commit: efad40d5723849a351ec700e8e4e346cac8c3454
Parents: ff03645
Author: rahul <ra...@knoldus.in>
Authored: Tue Jun 12 19:26:40 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Wed Jun 13 20:38:24 2018 +0530
----------------------------------------------------------------------
.../collector/impl/RawBasedResultCollector.java | 12 +++++++++---
.../sdv/generated/DataLoadingIUDTestCase.scala | 19 +++++++++++++++++++
2 files changed, 28 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/efad40d5/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
index d28df0a..7302b2c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
@@ -92,16 +92,22 @@ public class RawBasedResultCollector extends AbstractScannedResultCollector {
// re initialized with left over value
batchSize = 0;
}
+ // for every iteration of available rows filling newly created list of Object[] and add it to
+ // the final list so there is no mismatch in the counter while filling dimension and
+ // measure data
+ List<Object[]> collectedData = new ArrayList<>(availableBatchRowCount);
// fill dimension data
- fillDimensionData(scannedResult, listBasedResult, queryMeasures, availableBatchRowCount);
- fillMeasureData(scannedResult, listBasedResult);
+ fillDimensionData(scannedResult, collectedData, queryMeasures, availableBatchRowCount);
+ fillMeasureData(scannedResult, collectedData);
// increment the number of rows scanned in scanned result statistics
incrementScannedResultRowCounter(scannedResult, availableBatchRowCount);
// assign the left over rows to batch size if the number of rows fetched are lesser
// than batchSize
- if (listBasedResult.size() < availableBatchRowCount) {
+ if (collectedData.size() < availableBatchRowCount) {
batchSize += availableBatchRowCount - listBasedResult.size();
}
+ // add the collected data to the final list
+ listBasedResult.addAll(collectedData);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/efad40d5/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
index 4c232be..79458f5 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
@@ -3671,6 +3671,23 @@ test("HQ_Defect_TC_2016110901163", Include) {
sql(s"""drop table default.t_carbn01 """).collect
}
+ test("[CARBONDATA-2604] ", Include){
+ sql("drop table if exists brinjal").collect
+ sql("create table brinjal (imei string,AMSize string,channelsId string,ActiveCountry string, Activecity string,gamePointId double,deviceInformationId double,productionDate Timestamp,deliveryDate timestamp,deliverycharge double) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('table_blocksize'='2000','sort_columns'='imei')").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/vardhandaterestruct.csv' INTO TABLE brinjal OPTIONS('DELIMITER'=',', 'QUOTECHAR'= '','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'= 'imei,deviceInformationId,AMSize,channelsId,ActiveCountry,Activecity,gamePointId,productionDate,deliveryDate,deliverycharge')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/vardhandaterestruct.csv' INTO TABLE brinjal OPTIONS('DELIMITER'=',', 'QUOTECHAR'= '','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'= 'imei,deviceInformationId,AMSize,channelsId,ActiveCountry,Activecity,gamePointId,productionDate,deliveryDate,deliverycharge')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/vardhandaterestruct.csv' INTO TABLE brinjal OPTIONS('DELIMITER'=',', 'QUOTECHAR'= '','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'= 'imei,deviceInformationId,AMSize,channelsId,ActiveCountry,Activecity,gamePointId,productionDate,deliveryDate,deliverycharge')""").collect
+ sql("insert into brinjal select * from brinjal").collect
+ sql("update brinjal set (AMSize)= ('8RAM size') where AMSize='4RAM size'").collect
+ sql("delete from brinjal where AMSize='8RAM size'").collect
+ sql("delete from table brinjal where segment.id IN(0)").collect
+ sql("clean files for table brinjal").collect
+ sql("alter table brinjal compact 'minor'").collect
+ sql("alter table brinjal compact 'major'").collect
+ checkAnswer(s"""select count(*) from brinjal""",
+ Seq(Row(335)), "CARBONDATA-2604")
+ sql("drop table if exists brinjal")
+ }
override def afterAll {
sql("use default").collect
sql("drop table if exists t_carbn02").collect
@@ -3701,5 +3718,7 @@ override def afterAll {
sql("drop table if exists t_carbn01b").collect
sql("drop table if exists T_Hive1").collect
sql("drop table if exists T_Hive6").collect
+ sql("drop table if exists brinjal")
+
}
}
\ No newline at end of file
[22/50] [abbrv] carbondata git commit: [CARBONDATA-2309][DataLoad]
Add strategy to generate bigger carbondata files in case of small amout of
data
Posted by ja...@apache.org.
[CARBONDATA-2309][DataLoad] Add strategy to generate bigger carbondata files in case of small amout of data
This closes #2314
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/685087ed
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/685087ed
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/685087ed
Branch: refs/heads/carbonstore
Commit: 685087ed4de7ecc181de6ee43c9e5865eb26b650
Parents: 9b88a06
Author: ndwangsen <lu...@huawei.com>
Authored: Fri May 25 21:45:58 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jun 13 01:56:03 2018 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 12 ++
.../constants/CarbonLoadOptionConstants.java | 10 ++
.../carbondata/core/util/CarbonProperties.java | 12 ++
docs/useful-tips-on-carbondata.md | 3 +-
.../dataload/TestTableLoadMinSize.scala | 149 +++++++++++++++++++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 9 +-
.../management/CarbonLoadDataCommand.scala | 4 +-
.../loading/model/CarbonLoadModel.java | 15 ++
.../loading/model/CarbonLoadModelBuilder.java | 1 +
.../processing/loading/model/LoadOption.java | 2 +
.../processing/util/CarbonLoaderUtil.java | 105 +++++++++++--
.../processing/util/CarbonLoaderUtilTest.java | 51 ++++++-
.../scala/org/apache/spark/rpc/Master.scala | 3 +-
14 files changed, 355 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 08aa704..c7281dd 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1761,6 +1761,18 @@ public final class CarbonCommonConstants {
public static final String CARBON_LUCENE_INDEX_STOP_WORDS_DEFAULT = "false";
+ /**
+ * The node loads the smallest amount of data
+ */
+ @CarbonProperty
+ public static final String CARBON_LOAD_MIN_SIZE_INMB = "load_min_size_inmb";
+ public static final String CARBON_LOAD_MIN_NODE_SIZE_INMB_DEFAULT = "256";
+
+ /**
+ * the node minimum load data default value
+ */
+ public static final int CARBON_LOAD_MIN_SIZE_DEFAULT = 256;
+
private CarbonCommonConstants() {
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index a2213d5..6b8281c 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -151,4 +151,14 @@ public final class CarbonLoadOptionConstants {
public static final String CARBON_LOAD_SORT_MEMORY_SPILL_PERCENTAGE
= "carbon.load.sortMemory.spill.percentage";
public static final String CARBON_LOAD_SORT_MEMORY_SPILL_PERCENTAGE_DEFAULT = "0";
+
+ /**
+ * if loading data is too small, the original loading method will produce many small files.
+ * enable set the node load minimum amount of data,avoid producing many small files.
+ * This option is especially useful when you encounter a lot of small amounts of data.
+ */
+ @CarbonProperty
+ public static final String ENABLE_CARBON_LOAD_NODE_DATA_MIN_SIZE
+ = "carbon.load.min.size.enabled";
+ public static final String ENABLE_CARBON_LOAD_NODE_DATA_MIN_SIZE_DEFAULT = "false";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 4ee5199..6eb7de6 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -1310,6 +1310,18 @@ public final class CarbonProperties {
CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION_DEFAULT);
return skewedEnabled.equalsIgnoreCase("true");
}
+
+ /**
+ * whether optimization for the node loads the minimum amount of data is enabled
+ * @return true, if enabled; false for not enabled.
+ */
+ public boolean isLoadMinSizeOptimizationEnabled() {
+ String loadMinSize = getProperty(
+ CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_NODE_DATA_MIN_SIZE,
+ CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_NODE_DATA_MIN_SIZE_DEFAULT);
+ return loadMinSize.equalsIgnoreCase("true");
+ }
+
/**
* returns true if carbon property
* @param key
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/docs/useful-tips-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/useful-tips-on-carbondata.md b/docs/useful-tips-on-carbondata.md
index ff339d0..732d38f 100644
--- a/docs/useful-tips-on-carbondata.md
+++ b/docs/useful-tips-on-carbondata.md
@@ -170,5 +170,6 @@
| carbon.use.multiple.temp.dir | spark/carbonlib/carbon.properties | Data loading | Whether to use multiple YARN local directories during table data loading for disk load balance | After enabling 'carbon.use.local.dir', if this is set to true, CarbonData will use all YARN local directories during data load for disk load balance, that will improve the data load performance. Please enable this property when you encounter disk hotspot problem during data loading. |
| carbon.sort.temp.compressor | spark/carbonlib/carbon.properties | Data loading | Specify the name of compressor to compress the intermediate sort temporary files during sort procedure in data loading. | The optional values are 'SNAPPY','GZIP','BZIP2','LZ4' and empty. By default, empty means that Carbondata will not compress the sort temp files. This parameter will be useful if you encounter disk bottleneck. |
| carbon.load.skewedDataOptimization.enabled | spark/carbonlib/carbon.properties | Data loading | Whether to enable size based block allocation strategy for data loading. | When loading, carbondata will use file size based block allocation strategy for task distribution. It will make sure that all the executors process the same size of data -- It's useful if the size of your input data files varies widely, say 1MB~1GB. |
-
+ | carbon.load.min.size.enabled | spark/carbonlib/carbon.properties | Data loading | Whether to enable node minumun input data size allocation strategy for data loading.| When loading, carbondata will use node minumun input data size allocation strategy for task distribution. It will make sure the node load the minimum amount of data -- It's useful if the size of your input data files very small, say 1MB~256MB,Avoid generating a large number of small files. |
+
Note: If your CarbonData instance is provided only for query, you may specify the property 'spark.speculation=true' which is in conf directory of spark.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLoadMinSize.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLoadMinSize.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLoadMinSize.scala
new file mode 100644
index 0000000..ebb4e32
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLoadMinSize.scala
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.dataload
+
+import org.apache.spark.sql.Row
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.constants.CarbonLoadOptionConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.test.util.QueryTest
+
+/**
+ * Test Class for load_min_size
+ *
+ */
+
+class TestTableLoadMinSize extends QueryTest with BeforeAndAfterAll {
+ val testData1 = s"$resourcesPath/source.csv"
+
+ override def beforeAll {
+ sql("DROP TABLE IF EXISTS table_loadminsize1")
+ sql("DROP TABLE IF EXISTS table_loadminsize2")
+ sql("DROP TABLE IF EXISTS table_loadminsize3")
+ }
+
+ test("Value test: set table load min size in not int value") {
+ sql(
+ """
+ CREATE TABLE IF NOT EXISTS table_loadminsize1
+ (ID Int, date Timestamp, country String,
+ name String, phonetype String, serialname String, salary Int)
+ STORED BY 'org.apache.carbondata.format'
+ TBLPROPERTIES('table_blocksize'='128 MB')
+ """)
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_NODE_DATA_MIN_SIZE, "true")
+
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$testData1' into table table_loadminsize1 OPTIONS('load_min_size_inmb'='256 MB')
+ """)
+
+ checkAnswer(
+ sql("""
+ SELECT country, count(salary) AS amount
+ FROM table_loadminsize1
+ WHERE country IN ('china','france')
+ GROUP BY country
+ """),
+ Seq(Row("china", 96), Row("france", 1))
+ )
+ }
+
+ test("Function test:: set table load min size in int value") {
+
+ sql(
+ """
+ CREATE TABLE IF NOT EXISTS table_loadminsize2
+ (ID Int, date Timestamp, country String,
+ name String, phonetype String, serialname String, salary Int)
+ STORED BY 'org.apache.carbondata.format'
+ TBLPROPERTIES('table_blocksize'='128 MB')
+ """)
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_NODE_DATA_MIN_SIZE, "true")
+
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$testData1' into table table_loadminsize2 OPTIONS('load_min_size_inmb'='256')
+ """)
+
+ checkAnswer(
+ sql("""
+ SELECT country, count(salary) AS amount
+ FROM table_loadminsize2
+ WHERE country IN ('china','france')
+ GROUP BY country
+ """),
+ Seq(Row("china", 96), Row("france", 1))
+ )
+
+ }
+
+ test("Function test:: not set table load min size property") {
+
+ sql(
+ """
+ CREATE TABLE IF NOT EXISTS table_loadminsize3
+ (ID Int, date Timestamp, country String,
+ name String, phonetype String, serialname String, salary Int)
+ STORED BY 'org.apache.carbondata.format'
+ TBLPROPERTIES('table_blocksize'='128 MB')
+ """)
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_NODE_DATA_MIN_SIZE, "true")
+
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$testData1' into table table_loadminsize3
+ """)
+
+ checkAnswer(
+ sql("""
+ SELECT country, count(salary) AS amount
+ FROM table_loadminsize3
+ WHERE country IN ('china','france')
+ GROUP BY country
+ """),
+ Seq(Row("china", 96), Row("france", 1))
+ )
+
+ }
+
+
+ override def afterAll {
+ sql("DROP TABLE IF EXISTS table_loadminsize1")
+ sql("DROP TABLE IF EXISTS table_loadminsize2")
+ sql("DROP TABLE IF EXISTS table_loadminsize3")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ CarbonProperties.getInstance()
+ .addProperty(CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_NODE_DATA_MIN_SIZE, CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_NODE_DATA_MIN_SIZE_DEFAULT)
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 9bc5597..1f04fa4 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -885,7 +885,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
"ALL_DICTIONARY_PATH", "MAXCOLUMNS", "COMMENTCHAR", "DATEFORMAT", "BAD_RECORD_PATH",
"BATCH_SORT_SIZE_INMB", "GLOBAL_SORT_PARTITIONS", "SINGLE_PASS",
"IS_EMPTY_DATA_BAD_RECORD", "HEADER", "TIMESTAMPFORMAT", "SKIP_EMPTY_LINE",
- "SORT_COLUMN_BOUNDS"
+ "SORT_COLUMN_BOUNDS","LOAD_MIN_SIZE_INMB"
)
var isSupported = true
val invalidOptions = StringBuilder.newBuilder
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index bdbaef5..21a8641 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -1070,14 +1070,21 @@ object CarbonDataRDDFactory {
.ensureExecutorsAndGetNodeList(blockList, sqlContext.sparkContext)
val skewedDataOptimization = CarbonProperties.getInstance()
.isLoadSkewedDataOptimizationEnabled()
+ val loadMinSizeOptimization = CarbonProperties.getInstance()
+ .isLoadMinSizeOptimizationEnabled()
+ // get user ddl input the node loads the smallest amount of data
+ val expectedMinSizePerNode = carbonLoadModel.getLoadMinSize()
val blockAssignStrategy = if (skewedDataOptimization) {
CarbonLoaderUtil.BlockAssignmentStrategy.BLOCK_SIZE_FIRST
+ } else if (loadMinSizeOptimization) {
+ CarbonLoaderUtil.BlockAssignmentStrategy.NODE_MIN_SIZE_FIRST
} else {
CarbonLoaderUtil.BlockAssignmentStrategy.BLOCK_NUM_FIRST
}
LOGGER.info(s"Allocating block to nodes using strategy: $blockAssignStrategy")
+
val nodeBlockMapping = CarbonLoaderUtil.nodeBlockMapping(blockList.toSeq.asJava, -1,
- activeNodes.toList.asJava, blockAssignStrategy).asScala.toSeq
+ activeNodes.toList.asJava, blockAssignStrategy, expectedMinSizePerNode).asScala.toSeq
val timeElapsed: Long = System.currentTimeMillis - startTime
LOGGER.info("Total Time taken in block allocation: " + timeElapsed)
LOGGER.info(s"Total no of blocks: ${ blockList.length }, " +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index ba062c0..4703b23 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -24,7 +24,6 @@ import java.util.UUID
import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
-
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.conf.Configuration
import org.apache.spark.rdd.RDD
@@ -45,7 +44,6 @@ import org.apache.spark.sql.types._
import org.apache.spark.storage.StorageLevel
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.{CarbonReflectionUtils, CausedBy, FileUtils}
-
import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
@@ -76,7 +74,7 @@ import org.apache.carbondata.spark.dictionary.provider.SecureDictionaryServicePr
import org.apache.carbondata.spark.dictionary.server.SecureDictionaryServer
import org.apache.carbondata.spark.load.{CsvRDDHelper, DataLoadProcessorStepOnSpark}
import org.apache.carbondata.spark.rdd.CarbonDataRDDFactory
-import org.apache.carbondata.spark.util.{CarbonScalaUtil, GlobalDictionaryUtil, SparkDataTypeConverterImpl}
+import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, GlobalDictionaryUtil, SparkDataTypeConverterImpl}
case class CarbonLoadDataCommand(
databaseNameOp: Option[String],
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index f82de83..f267fa7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -210,6 +210,11 @@ public class CarbonLoadModel implements Serializable {
* Flder path to where data should be written for this load.
*/
private String dataWritePath;
+
+ /**
+ * sort columns bounds
+ */
+ private String loadMinSize;
private List<String> mergedSegmentIds;
@@ -388,6 +393,14 @@ public class CarbonLoadModel implements Serializable {
this.sortColumnsBoundsStr = sortColumnsBoundsStr;
}
+ public String getLoadMinSize() {
+ return loadMinSize;
+ }
+
+ public void setLoadMinSize(String loadMinSize) {
+ this.loadMinSize = loadMinSize;
+ }
+
/**
* Get copy with taskNo.
* Broadcast value is shared in process, so we need to copy it to make sure the value in each
@@ -439,6 +452,7 @@ public class CarbonLoadModel implements Serializable {
copy.badRecordsLocation = badRecordsLocation;
copy.isLoadWithoutConverterStep = isLoadWithoutConverterStep;
copy.sortColumnsBoundsStr = sortColumnsBoundsStr;
+ copy.loadMinSize = loadMinSize;
return copy;
}
@@ -492,6 +506,7 @@ public class CarbonLoadModel implements Serializable {
copyObj.badRecordsLocation = badRecordsLocation;
copyObj.isAggLoadRequest = isAggLoadRequest;
copyObj.sortColumnsBoundsStr = sortColumnsBoundsStr;
+ copyObj.loadMinSize = loadMinSize;
return copyObj;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
index 9a9d09e..4ad1984 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
@@ -270,6 +270,7 @@ public class CarbonLoadModelBuilder {
carbonLoadModel.setMaxColumns(String.valueOf(validatedMaxColumns));
carbonLoadModel.readAndSetLoadMetadataDetails();
carbonLoadModel.setSortColumnsBoundsStr(optionsFinal.get("sort_column_bounds"));
+ carbonLoadModel.setLoadMinSize(optionsFinal.get(CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB));
}
private int validateMaxColumns(String[] csvHeaders, String maxColumns)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
index 4ff1cce..1a65937 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
@@ -196,6 +196,8 @@ public class LoadOption {
optionsFinal.put("single_pass", String.valueOf(singlePass));
optionsFinal.put("sort_scope", "local_sort");
optionsFinal.put("sort_column_bounds", Maps.getOrDefault(options, "sort_column_bounds", ""));
+ optionsFinal.put(CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB, Maps.getOrDefault(options,CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB, CarbonCommonConstants
+ .CARBON_LOAD_MIN_NODE_SIZE_INMB_DEFAULT));
return optionsFinal;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 6d938e1..d5a0b78 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -53,6 +53,7 @@ import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.CarbonIndexFileMergeWriter;
@@ -77,7 +78,8 @@ public final class CarbonLoaderUtil {
*/
public enum BlockAssignmentStrategy {
BLOCK_NUM_FIRST("Assign blocks to node base on number of blocks"),
- BLOCK_SIZE_FIRST("Assign blocks to node base on data size of blocks");
+ BLOCK_SIZE_FIRST("Assign blocks to node base on data size of blocks"),
+ NODE_MIN_SIZE_FIRST("Assign blocks to node base on minumun size of inputs");
private String name;
BlockAssignmentStrategy(String name) {
this.name = name;
@@ -537,7 +539,7 @@ public final class CarbonLoaderUtil {
List<String> activeNode) {
Map<String, List<Distributable>> mapOfNodes =
CarbonLoaderUtil.nodeBlockMapping(blockInfos, noOfNodesInput, activeNode,
- BlockAssignmentStrategy.BLOCK_NUM_FIRST);
+ BlockAssignmentStrategy.BLOCK_NUM_FIRST, null);
int taskPerNode = parallelism / mapOfNodes.size();
//assigning non zero value to noOfTasksPerNode
int noOfTasksPerNode = taskPerNode == 0 ? 1 : taskPerNode;
@@ -554,7 +556,7 @@ public final class CarbonLoaderUtil {
public static Map<String, List<Distributable>> nodeBlockMapping(List<Distributable> blockInfos,
int noOfNodesInput) {
return nodeBlockMapping(blockInfos, noOfNodesInput, null,
- BlockAssignmentStrategy.BLOCK_NUM_FIRST);
+ BlockAssignmentStrategy.BLOCK_NUM_FIRST,null);
}
/**
@@ -575,11 +577,12 @@ public final class CarbonLoaderUtil {
* @param noOfNodesInput -1 if number of nodes has to be decided
* based on block location information
* @param blockAssignmentStrategy strategy used to assign blocks
+ * @param loadMinSize the property load_min_size_inmb specified by the user
* @return a map that maps node to blocks
*/
public static Map<String, List<Distributable>> nodeBlockMapping(
List<Distributable> blockInfos, int noOfNodesInput, List<String> activeNodes,
- BlockAssignmentStrategy blockAssignmentStrategy) {
+ BlockAssignmentStrategy blockAssignmentStrategy, String expectedMinSizePerNode ) {
ArrayList<NodeMultiBlockRelation> rtnNode2Blocks = new ArrayList<>();
Set<Distributable> uniqueBlocks = new HashSet<>(blockInfos);
@@ -596,20 +599,52 @@ public final class CarbonLoaderUtil {
// calculate the average expected size for each node
long sizePerNode = 0;
+ long totalFileSize = 0;
if (BlockAssignmentStrategy.BLOCK_NUM_FIRST == blockAssignmentStrategy) {
sizePerNode = blockInfos.size() / noofNodes;
sizePerNode = sizePerNode <= 0 ? 1 : sizePerNode;
- } else if (BlockAssignmentStrategy.BLOCK_SIZE_FIRST == blockAssignmentStrategy) {
- long totalFileSize = 0;
+ } else if (BlockAssignmentStrategy.BLOCK_SIZE_FIRST == blockAssignmentStrategy
+ || BlockAssignmentStrategy.NODE_MIN_SIZE_FIRST == blockAssignmentStrategy) {
for (Distributable blockInfo : uniqueBlocks) {
totalFileSize += ((TableBlockInfo) blockInfo).getBlockLength();
}
sizePerNode = totalFileSize / noofNodes;
}
- // assign blocks to each node
- assignBlocksByDataLocality(rtnNode2Blocks, sizePerNode, uniqueBlocks, originNode2Blocks,
- activeNodes, blockAssignmentStrategy);
+ // if enable to control the minimum amount of input data for each node
+ if (BlockAssignmentStrategy.NODE_MIN_SIZE_FIRST == blockAssignmentStrategy) {
+ long iexpectedMinSizePerNode = 0;
+ // validate the property load_min_size_inmb specified by the user
+ if (CarbonUtil.validateValidIntType(expectedMinSizePerNode)) {
+ iexpectedMinSizePerNode = Integer.parseInt(expectedMinSizePerNode);
+ } else {
+ LOGGER.warn("Invalid load_min_size_inmb value found: " + expectedMinSizePerNode
+ + ", only int value greater than 0 is supported.");
+ iexpectedMinSizePerNode = CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_DEFAULT;
+ }
+ // If the average expected size for each node greater than load min size,
+ // then fall back to default strategy
+ if (iexpectedMinSizePerNode * 1024 * 1024 < sizePerNode) {
+ if (CarbonProperties.getInstance().isLoadSkewedDataOptimizationEnabled()) {
+ blockAssignmentStrategy = BlockAssignmentStrategy.BLOCK_SIZE_FIRST;
+ } else {
+ blockAssignmentStrategy = BlockAssignmentStrategy.BLOCK_NUM_FIRST;
+ }
+ LOGGER.info("Specified minimum data size to load is less than the average size for each node, "
+ + "fallback to default strategy" + blockAssignmentStrategy);
+ } else {
+ sizePerNode = iexpectedMinSizePerNode;
+ }
+ }
+
+ if (BlockAssignmentStrategy.NODE_MIN_SIZE_FIRST == blockAssignmentStrategy) {
+ // assign blocks to each node ignore data locality
+ assignBlocksIgnoreDataLocality(rtnNode2Blocks, sizePerNode, uniqueBlocks, activeNodes);
+ } else {
+ // assign blocks to each node
+ assignBlocksByDataLocality(rtnNode2Blocks, sizePerNode, uniqueBlocks, originNode2Blocks,
+ activeNodes, blockAssignmentStrategy);
+ }
// if any blocks remain then assign them to nodes in round robin.
assignLeftOverBlocks(rtnNode2Blocks, uniqueBlocks, sizePerNode, activeNodes,
@@ -623,7 +658,7 @@ public final class CarbonLoaderUtil {
}
return rtnNodeBlocksMap;
}
-
+
/**
* Assigning the blocks of a node to tasks.
*
@@ -757,6 +792,7 @@ public final class CarbonLoaderUtil {
populateBlocksByNum(remainingBlocks, expectedSizePerNode, blockLst);
break;
case BLOCK_SIZE_FIRST:
+ case NODE_MIN_SIZE_FIRST:
populateBlocksBySize(remainingBlocks, expectedSizePerNode, blockLst);
break;
default:
@@ -836,6 +872,7 @@ public final class CarbonLoaderUtil {
roundRobinAssignBlocksByNum(node2Blocks, remainingBlocks);
break;
case BLOCK_SIZE_FIRST:
+ case NODE_MIN_SIZE_FIRST:
roundRobinAssignBlocksBySize(node2Blocks, remainingBlocks);
break;
default:
@@ -983,6 +1020,54 @@ public final class CarbonLoaderUtil {
}
/**
+ * allocate distributable blocks to nodes based on ignore data locality
+ */
+ private static void assignBlocksIgnoreDataLocality(
+ ArrayList<NodeMultiBlockRelation> outputNode2Blocks,
+ long expectedSizePerNode, Set<Distributable> remainingBlocks,
+ List<String> activeNodes) {
+ // get all blocks
+ Set<Distributable> uniqueBlocks = new HashSet<>(remainingBlocks);
+ // shuffle activeNodes ignore data locality
+ List<String> shuffleNodes = new ArrayList<>(activeNodes);
+ Collections.shuffle(shuffleNodes);
+
+ for (String activeNode : shuffleNodes) {
+ long nodeCapacity = 0;
+ NodeMultiBlockRelation nodeBlock = new NodeMultiBlockRelation(activeNode,
+ new ArrayList<Distributable>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE));
+ // loop thru blocks of each Node
+ for (Distributable block : uniqueBlocks) {
+ if (!remainingBlocks.contains(block)) {
+ // this block has been added before
+ continue;
+ }
+
+ long thisBlockSize = ((TableBlockInfo) block).getBlockLength();
+ if (nodeCapacity == 0
+ || nodeCapacity + thisBlockSize <= expectedSizePerNode * 1024 * 1024) {
+ nodeBlock.getBlocks().add(block);
+ nodeCapacity += thisBlockSize;
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "First Assignment iteration: " + ((TableBlockInfo) block).getFilePath() + '-'
+ + ((TableBlockInfo) block).getBlockLength() + "-->" + activeNode);
+ }
+ remainingBlocks.remove(block);
+ // this block is too big for current node and there are still capacity left
+ // for small files, so continue to allocate block on this node in next iteration.
+ } else {
+ // No need to continue loop as node is full
+ break;
+ }
+ }
+ if (nodeBlock.getBlocks().size() != 0) {
+ outputNode2Blocks.add(nodeBlock);
+ }
+ }
+ }
+
+ /**
* method validates whether the node is active or not.
*
* @param activeNode
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/processing/src/test/java/org/apache/carbondata/processing/util/CarbonLoaderUtilTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/util/CarbonLoaderUtilTest.java b/processing/src/test/java/org/apache/carbondata/processing/util/CarbonLoaderUtilTest.java
index 9c66ada..94f8b84 100644
--- a/processing/src/test/java/org/apache/carbondata/processing/util/CarbonLoaderUtilTest.java
+++ b/processing/src/test/java/org/apache/carbondata/processing/util/CarbonLoaderUtilTest.java
@@ -72,6 +72,29 @@ public class CarbonLoaderUtilTest {
return blockInfos;
}
+ private List<Distributable> generateBlocks2() {
+ List<Distributable> blockInfos = new ArrayList<>();
+ String filePath = "/fakepath";
+ String blockId = "1";
+
+ String[] locations = new String[] { "host2", "host3" };
+ ColumnarFormatVersion version = ColumnarFormatVersion.V1;
+
+ TableBlockInfo tableBlockInfo1 = new TableBlockInfo(filePath + "_a", 0,
+ blockId, locations, 30 * 1024 * 1024, version, null);
+ blockInfos.add(tableBlockInfo1);
+
+ TableBlockInfo tableBlockInfo2 = new TableBlockInfo(filePath + "_b", 0,
+ blockId, locations, 30 * 1024 * 1024, version, null);
+ blockInfos.add(tableBlockInfo2);
+
+ TableBlockInfo tableBlockInfo3 = new TableBlockInfo(filePath + "_c", 0,
+ blockId, locations, 30 * 1024 * 1024, version, null);
+ blockInfos.add(tableBlockInfo3);
+
+ return blockInfos;
+ }
+
private List<String> generateExecutors() {
List<String> activeNodes = new ArrayList<>();
activeNodes.add("host1");
@@ -86,9 +109,9 @@ public class CarbonLoaderUtilTest {
List<String> activeNodes = generateExecutors();
// the blocks are assigned by size, so the number of block for each node are different
- Map<String, List<Distributable>> nodeMappingBySize =
- CarbonLoaderUtil.nodeBlockMapping(blockInfos, -1, activeNodes,
- CarbonLoaderUtil.BlockAssignmentStrategy.BLOCK_SIZE_FIRST);
+ Map<String, List<Distributable>> nodeMappingBySize = CarbonLoaderUtil
+ .nodeBlockMapping(blockInfos, -1, activeNodes,
+ CarbonLoaderUtil.BlockAssignmentStrategy.BLOCK_SIZE_FIRST, null);
LOGGER.info(convertMapListAsString(nodeMappingBySize));
Assert.assertEquals(3, nodeMappingBySize.size());
for (Map.Entry<String, List<Distributable>> entry : nodeMappingBySize.entrySet()) {
@@ -102,9 +125,9 @@ public class CarbonLoaderUtilTest {
}
// the blocks are assigned by number, so the number of blocks for each node are nearly the same
- Map<String, List<Distributable>> nodeMappingByNum =
- CarbonLoaderUtil.nodeBlockMapping(blockInfos, -1, activeNodes,
- CarbonLoaderUtil.BlockAssignmentStrategy.BLOCK_NUM_FIRST);
+ Map<String, List<Distributable>> nodeMappingByNum = CarbonLoaderUtil
+ .nodeBlockMapping(blockInfos, -1, activeNodes,
+ CarbonLoaderUtil.BlockAssignmentStrategy.BLOCK_NUM_FIRST, null);
LOGGER.info(convertMapListAsString(nodeMappingByNum));
Assert.assertEquals(3, nodeMappingBySize.size());
for (Map.Entry<String, List<Distributable>> entry : nodeMappingByNum.entrySet()) {
@@ -113,6 +136,22 @@ public class CarbonLoaderUtilTest {
}
}
+ @Test
+ public void testNodeBlockMappingByNodeRandom() throws Exception {
+ List<Distributable> blockInfos = generateBlocks2();
+ List<String> activeNodes = generateExecutors();
+
+ // the blocks are assigned by node as random, The node loads the smallest amount of data by user specified
+ Map<String, List<Distributable>> nodeMappingByRandom = CarbonLoaderUtil
+ .nodeBlockMapping(blockInfos, -1, activeNodes,
+ CarbonLoaderUtil.BlockAssignmentStrategy.NODE_MIN_SIZE_FIRST, "90");
+ LOGGER.info(convertMapListAsString(nodeMappingByRandom));
+ Assert.assertEquals(1, nodeMappingByRandom.size());
+ for (Map.Entry<String, List<Distributable>> entry : nodeMappingByRandom.entrySet()) {
+ Assert.assertTrue(entry.getValue().size() == blockInfos.size());
+ }
+ }
+
private <K, T> String convertMapListAsString(Map<K, List<T>> mapList) {
StringBuffer sb = new StringBuffer();
for (Map.Entry<K, List<T>> entry : mapList.entrySet()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/store/search/src/main/scala/org/apache/spark/rpc/Master.scala
----------------------------------------------------------------------
diff --git a/store/search/src/main/scala/org/apache/spark/rpc/Master.scala b/store/search/src/main/scala/org/apache/spark/rpc/Master.scala
index f48f5e4..b7630fb 100644
--- a/store/search/src/main/scala/org/apache/spark/rpc/Master.scala
+++ b/store/search/src/main/scala/org/apache/spark/rpc/Master.scala
@@ -279,7 +279,8 @@ class Master(sparkConf: SparkConf) {
distributables.asJava,
-1,
getWorkers.asJava,
- CarbonLoaderUtil.BlockAssignmentStrategy.BLOCK_NUM_FIRST)
+ CarbonLoaderUtil.BlockAssignmentStrategy.BLOCK_NUM_FIRST,
+ null)
}
/** return hostname of all workers */
[37/50] [abbrv] carbondata git commit: [CARBONDATA-2611] Added Test
Cases for Local Dictionary Support for Create Table comand
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c5a4ec07/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 65ff76d..9af8817 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -321,9 +321,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
case scala.util.Failure(ex) =>
LOGGER
.debug(
- "invalid value is configured for local_dictionary_threshold, considering the defaut" +
- " " +
- "value")
+ "invalid value is configured for local_dictionary_threshold, considering the " +
+ "default value")
tableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD,
CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT)
}
@@ -362,9 +361,11 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
.distinct.length !=
List(localDictIncludeCols, localDictExcludeCols).mkString(",").split(",")
.length) {
+ val duplicateColumns = localDictIncludeCols.diff(localDictExcludeCols.distinct).distinct
val errMsg =
- "Column ambiguity as duplicate columns present in LOCAL_DICTIONARY_INCLUDE and " +
- "LOCAL_DICTIONARY_INCLUDE.Duplicate columns are not allowed."
+ "Column ambiguity as duplicate column(s): " +
+ duplicateColumns.mkString("(", ",", ")") + "are present in LOCAL_DICTIONARY_INCLUDE " +
+ "and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed."
throw new MalformedCarbonCommandException(errMsg)
}
}
@@ -411,10 +412,11 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
// check if the duplicate columns are specified in table schema
if (localDictColumns.distinct.lengthCompare(localDictColumns.size) != 0) {
- val a = localDictColumns.diff(localDictColumns.distinct).distinct
- val errMsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE contains Duplicate Columns " +
- a.mkString("(", ",", ")") +
- ". Please check create table statement."
+ val duplicateColumns = localDictColumns.diff(localDictColumns.distinct).distinct
+ val errMsg =
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE contains Duplicate Columns: " +
+ duplicateColumns.mkString("(", ",", ")") +
+ ". Please check create table statement."
throw new MalformedCarbonCommandException(errMsg)
}
@@ -436,8 +438,9 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
!x.dataType.get.equalsIgnoreCase("ARRAY"))) {
val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " +
dictColm.trim +
- " is not a String datatype column. LOCAL_DICTIONARY_COLUMN should be no " +
- "dictionary string datatype column.Please check create table statement."
+ " is not a String/complex datatype column. LOCAL_DICTIONARY_COLUMN should " +
+ "be no dictionary string/complex datatype column.Please check create table" +
+ " statement."
throw new MalformedCarbonCommandException(errormsg)
}
}
@@ -448,10 +451,12 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
tableProperties(CarbonCommonConstants.DICTIONARY_INCLUDE).split(",").map(_.trim)
localDictColumns.foreach { distCol =>
if (dictIncludeColumns.exists(x => x.equalsIgnoreCase(distCol.trim))) {
+ val duplicateColumns = dictIncludeColumns.diff(localDictColumns.distinct).distinct
val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " +
- distCol.trim +
+ duplicateColumns.mkString("(", ",", ")") +
" specified in Dictionary include. Local Dictionary will not be " +
- "generated for Dictionary include. Please check create table statement."
+ "generated for Dictionary include columns. Please check create table " +
+ "statement."
throw new MalformedCarbonCommandException(errormsg)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c5a4ec07/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
index c6bd567..3b56a35 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
@@ -141,7 +141,7 @@ private[sql] case class CarbonDescribeFormattedCommand(
*/
def getDictColumnString(localDictColumns: Array[String]): String = {
val dictColumns: StringBuilder = new StringBuilder
- localDictColumns.foreach(column => dictColumns.append(column).append(","))
+ localDictColumns.foreach(column => dictColumns.append(column.trim).append(","))
dictColumns.toString().patch(dictColumns.toString().lastIndexOf(","), "", 1)
}
[09/50] [abbrv] carbondata git commit: [CARBONDATA-2571] Calculating
the carbonindex and carbondata file size of a table is wrong
Posted by ja...@apache.org.
[CARBONDATA-2571] Calculating the carbonindex and carbondata file size of a table is wrong
Problem:
While calculating the carbonindex files size, we are checking either index file or merge file. But in PR#2333, implementation is changed to fill both
the file name and the merge file name. So, we have to consider both fields.
Solution:
While calculating the carbonindex files size, we have to consider both the files and mergeFileName fields. We should get the list of index files from
these 2 fields and then calculate the size of the files.
This closes #2358
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/27d70599
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/27d70599
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/27d70599
Branch: refs/heads/carbonstore
Commit: 27d7059984962b97bcaf576fed496653932ea743
Parents: 92d9b92
Author: dhatchayani <dh...@gmail.com>
Authored: Fri Jun 1 15:13:38 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Tue Jun 5 11:51:52 2018 +0530
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 37 +++++++++++---------
1 file changed, 20 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/27d70599/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 1526047..5a7bce3 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2688,27 +2688,30 @@ public final class CarbonUtil {
throws IOException {
long carbonDataSize = 0L;
long carbonIndexSize = 0L;
- List<String> listOfFilesRead = new ArrayList<>();
HashMap<String, Long> dataAndIndexSize = new HashMap<String, Long>();
- if (fileStore.getLocationMap() != null) {
+ Map<String, SegmentFileStore.FolderDetails> locationMap = fileStore.getLocationMap();
+ if (locationMap != null) {
fileStore.readIndexFiles();
- Map<String, String> indexFiles = fileStore.getIndexFiles();
Map<String, List<String>> indexFilesMap = fileStore.getIndexFilesMap();
- for (Map.Entry<String, List<String>> entry : indexFilesMap.entrySet()) {
- // get the size of carbonindex file
- String indexFile = entry.getKey();
- String mergeIndexFile = indexFiles.get(indexFile);
- if (null != mergeIndexFile) {
- String mergeIndexPath = indexFile
- .substring(0, indexFile.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR) + 1)
- + mergeIndexFile;
- if (!listOfFilesRead.contains(mergeIndexPath)) {
- carbonIndexSize += FileFactory.getCarbonFile(mergeIndexPath).getSize();
- listOfFilesRead.add(mergeIndexPath);
- }
- } else {
- carbonIndexSize += FileFactory.getCarbonFile(indexFile).getSize();
+ // get the size of carbonindex file
+ for (Map.Entry<String, SegmentFileStore.FolderDetails> entry : locationMap.entrySet()) {
+ SegmentFileStore.FolderDetails folderDetails = entry.getValue();
+ Set<String> carbonindexFiles = folderDetails.getFiles();
+ String mergeFileName = folderDetails.getMergeFileName();
+ if (null != mergeFileName) {
+ String mergeIndexPath =
+ fileStore.getTablePath() + entry.getKey() + CarbonCommonConstants.FILE_SEPARATOR
+ + mergeFileName;
+ carbonIndexSize += FileFactory.getCarbonFile(mergeIndexPath).getSize();
}
+ for (String indexFile : carbonindexFiles) {
+ String indexPath =
+ fileStore.getTablePath() + entry.getKey() + CarbonCommonConstants.FILE_SEPARATOR
+ + indexFile;
+ carbonIndexSize += FileFactory.getCarbonFile(indexPath).getSize();
+ }
+ }
+ for (Map.Entry<String, List<String>> entry : indexFilesMap.entrySet()) {
// get the size of carbondata files
for (String blockFile : entry.getValue()) {
carbonDataSize += FileFactory.getCarbonFile(blockFile).getSize();
[17/50] [abbrv] carbondata git commit: [CARBONDATA-2573] integrate
carbonstore mv branch
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestTPCDS_1_4_Batch.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestTPCDS_1_4_Batch.scala b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestTPCDS_1_4_Batch.scala
index 074bf00..e564052 100644
--- a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestTPCDS_1_4_Batch.scala
+++ b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestTPCDS_1_4_Batch.scala
@@ -1,20 +1,3 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
package org.apache.carbondata.mv.rewrite.matching
object TestTPCDS_1_4_Batch {
@@ -32,73 +15,67 @@ object TestTPCDS_1_4_Batch {
|FROM item
|WHERE i_item_sk IN (2, 3, 5, 7, 11, 13, 17, 19)
""".stripMargin.trim,
- """
- |SELECT
- |FROM
- |WHERE
- """.stripMargin.trim),
+ """
+ """.stripMargin.trim),
// test case 2: test SELECT-SELECT-EXACT_MATCH with SELECT containing join (derive from q64)
- ("case_2",
- """
- |SELECT cs1.product_name, cs1.store_name, cs1.store_zip, cs1.b_street_number,
- | cs1.b_streen_name, cs1.b_city, cs1.b_zip, cs1.c_street_number, cs1.c_street_name,
- | cs1.c_city, cs1.c_zip, cs1.syear, cs1.cnt, cs1.s1, cs1.s2, cs1.s3, cs2.s1,
- | cs2.s2, cs2.s3, cs2.syear, cs2.cnt
- |FROM cross_sales cs1,cross_sales cs2
- |WHERE cs1.item_sk=cs2.item_sk AND
- | cs1.syear = 1999 AND
- | cs2.syear = 1999 + 1 AND
- | cs2.cnt <= cs1.cnt AND
- | cs1.store_name = cs2.store_name AND
- | cs1.store_zip = cs2.store_zip
- """.stripMargin.trim,
- """
- |SELECT cs1.product_name, cs1.store_name, cs1.store_zip, cs1.b_street_number,
- | cs1.b_streen_name, cs1.b_city, cs1.b_zip, cs1.c_street_number, cs1.c_street_name,
- | cs1.c_city, cs1.c_zip, cs1.syear, cs1.cnt, cs1.s1, cs1.s2, cs1.s3, cs2.s1,
- | cs2.s2, cs2.s3
- |FROM cross_sales cs1,cross_sales cs2
- |WHERE cs1.item_sk=cs2.item_sk AND
- | cs1.syear = 1999 AND
- | cs2.syear = 1999 + 1 AND
- | cs2.cnt <= cs1.cnt AND
- | cs1.store_name = cs2.store_name AND
- | cs1.store_zip = cs2.store_zip
- |ORDER BY cs1.product_name, cs1.store_name, cs2.cnt
- """.stripMargin.trim,
- """
- |SELECT
- |FROM
- |WHERE
- """.stripMargin.trim),
+ // cross_sales not in Tpcds_1_4_Tables.scala
+// ("case_2",
+// """
+// |SELECT cs1.product_name, cs1.store_name, cs1.store_zip, cs1.b_street_number,
+// | cs1.b_streen_name, cs1.b_city, cs1.b_zip, cs1.c_street_number, cs1.c_street_name,
+// | cs1.c_city, cs1.c_zip, cs1.syear, cs1.cnt, cs1.s1, cs1.s2, cs1.s3, cs2.s1,
+// | cs2.s2, cs2.s3, cs2.syear, cs2.cnt
+// |FROM cross_sales cs1,cross_sales cs2
+// |WHERE cs1.item_sk=cs2.item_sk AND
+// | cs1.syear = 1999 AND
+// | cs2.syear = 1999 + 1 AND
+// | cs2.cnt <= cs1.cnt AND
+// | cs1.store_name = cs2.store_name AND
+// | cs1.store_zip = cs2.store_zip
+// """.stripMargin.trim,
+// """
+// |SELECT cs1.product_name, cs1.store_name, cs1.store_zip, cs1.b_street_number,
+// | cs1.b_streen_name, cs1.b_city, cs1.b_zip, cs1.c_street_number, cs1.c_street_name,
+// | cs1.c_city, cs1.c_zip, cs1.syear, cs1.cnt, cs1.s1, cs1.s2, cs1.s3, cs2.s1,
+// | cs2.s2, cs2.s3
+// |FROM cross_sales cs1,cross_sales cs2
+// |WHERE cs1.item_sk=cs2.item_sk AND
+// | cs1.syear = 1999 AND
+// | cs2.syear = 1999 + 1 AND
+// | cs2.cnt <= cs1.cnt AND
+// | cs1.store_name = cs2.store_name AND
+// | cs1.store_zip = cs2.store_zip
+// |ORDER BY cs1.product_name, cs1.store_name, cs2.cnt
+// """.stripMargin.trim,
+// """
+// |SELECT
+// |FROM
+// |WHERE
+// """.stripMargin.trim),
// test case 3: test simple SELECT with GROUPBY (from q99)
("case_3",
"""
|SELECT count(ss_sold_date_sk) as not_null_total,
| max(ss_sold_date_sk) as max_ss_sold_date_sk,
| max(ss_sold_time_sk) as max_ss_sold_time_sk,
- | ss_item_sk,
- | ss_store_sk
+ | ss_item_sk, ss_store_sk
|FROM store_sales
|GROUP BY ss_item_sk, ss_store_sk
""".stripMargin.trim,
"""
|SELECT count(ss_sold_date_sk) as not_null_total,
| max(ss_sold_date_sk) as max_ss_sold_date_sk,
- | ss_item_sk,
- | ss_store_sk
+ | ss_item_sk, ss_store_sk
|FROM store_sales
|GROUP BY ss_item_sk, ss_store_sk
""".stripMargin.trim,
"""
- |SELECT gen_subsumer_0.`not_null_total`,
- | gen_subsumer_0.`max_ss_sold_date_sk`,
- | gen_subsumer_0.`ss_item_sk`,
- | gen_subsumer_0.`ss_store_sk`
+ |SELECT gen_subsumer_0.`not_null_total` AS `not_null_total`, gen_subsumer_0.`max_ss_sold_date_sk` AS `max_ss_sold_date_sk`, gen_subsumer_0.`ss_item_sk`, gen_subsumer_0.`ss_store_sk`
|FROM
- | (SELECT count(`ss_sold_date_sk`) AS `not_null_total`, max(`ss_sold_date_sk`) AS `max_ss_sold_date_sk`, max(`ss_sold_time_sk`) AS `max_ss_sold_time_sk`, `ss_item_sk`, `ss_store_sk`
- | FROM store_sales
- | GROUP BY `ss_item_sk`, `ss_store_sk`) gen_subsumer_0
+ | (SELECT count(store_sales.`ss_sold_date_sk`) AS `not_null_total`, max(store_sales.`ss_sold_date_sk`) AS `max_ss_sold_date_sk`, max(store_sales.`ss_sold_time_sk`) AS `max_ss_sold_time_sk`, store_sales.`ss_item_sk`, store_sales.`ss_store_sk`
+ | FROM
+ | store_sales
+ | GROUP BY store_sales.`ss_item_sk`, store_sales.`ss_store_sk`) gen_subsumer_0
""".stripMargin.trim),
// test case 4 test SELECT containing join with GROUPBY (from q65)
("case_4",
@@ -115,11 +92,11 @@ object TestTPCDS_1_4_Batch {
|GROUP BY ss_store_sk, ss_item_sk
""".stripMargin.trim,
"""
- |SELECT `ss_store_sk`, `ss_item_sk`, sum(`ss_sales_price`) AS `revenue`
+ |SELECT store_sales.`ss_store_sk`, store_sales.`ss_item_sk`, sum(store_sales.`ss_sales_price`) AS `revenue`
|FROM
| store_sales
- | INNER JOIN date_dim ON (`d_month_seq` >= 1176) AND (`d_month_seq` <= 1187) AND (`ss_sold_date_sk` = `d_date_sk`)
- |GROUP BY `ss_store_sk`, `ss_item_sk`
+ | INNER JOIN date_dim ON (date_dim.`d_month_seq` >= 1176) AND (date_dim.`d_month_seq` <= 1187) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
+ |GROUP BY store_sales.`ss_store_sk`, store_sales.`ss_item_sk`
""".stripMargin.trim),
// the following 6 cases involve an MV of store_sales, item, date_dim
// q3
@@ -152,12 +129,12 @@ object TestTPCDS_1_4_Batch {
"""
|SELECT gen_subsumer_0.`d_year`, gen_subsumer_0.`i_brand_id` AS `brand_id`, gen_subsumer_0.`i_brand` AS `brand`, sum(gen_subsumer_0.`sum_agg`) AS `sum_agg`
|FROM
- | (SELECT `d_date`, `d_moy`, `d_year`, `i_brand`, `i_brand_id`, `i_item_id`, `i_item_desc`, `i_manufact_id`, substring(`i_item_desc`, 1, 30) AS `itemdesc`, `i_category`, `i_class`, `i_current_price`, `i_item_sk`, `ss_store_sk`, sum(`ss_ext_sales_price`) AS `sum_agg`, sum((CAST(CAST(`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(`ss_list_price` AS DECIMAL(12,2)))) AS `sales`, count(1) AS `number_sales`
+ | (SELECT dt.`d_date`, dt.`d_moy`, dt.`d_year`, item.`i_brand`, item.`i_brand_id`, item.`i_item_id`, item.`i_item_desc`, item.`i_manufact_id`, substring(item.`i_item_desc`, 1, 30) AS `itemdesc`, item.`i_category`, item.`i_class`, item.`i_current_price`, item.`i_item_sk`, store_sales.`ss_store_sk`, sum(store_sales.`ss_ext_sales_price`) AS `sum_agg`, sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_list_price` AS DECIMAL(12,2)))) AS `sales`, count(1) AS `number_sales`
| FROM
| date_dim dt
- | INNER JOIN store_sales ON (`d_date_sk` = `ss_sold_date_sk`)
- | INNER JOIN item ON (`ss_item_sk` = `i_item_sk`)
- | GROUP BY `d_date`, `d_moy`, `d_year`, `i_brand`, `i_brand_id`, `i_item_id`, `i_item_desc`, `i_manufact_id`, substring(`i_item_desc`, 1, 30), `i_category`, `i_category_id`, `i_class`, `i_class_id`, `i_current_price`, `i_manager_id`, `i_item_sk`, `ss_store_sk`) gen_subsumer_0
+ | INNER JOIN store_sales ON (dt.`d_date_sk` = store_sales.`ss_sold_date_sk`)
+ | INNER JOIN item ON (store_sales.`ss_item_sk` = item.`i_item_sk`)
+ | GROUP BY dt.`d_date`, dt.`d_moy`, dt.`d_year`, item.`i_brand`, item.`i_brand_id`, item.`i_item_id`, item.`i_item_desc`, item.`i_manufact_id`, substring(item.`i_item_desc`, 1, 30), item.`i_category`, item.`i_category_id`, item.`i_class`, item.`i_class_id`, item.`i_current_price`, item.`i_manager_id`, item.`i_item_sk`, store_sales.`ss_store_sk`) gen_subsumer_0
|WHERE
| (gen_subsumer_0.`d_moy` = 11) AND (gen_subsumer_0.`i_manufact_id` = 128)
|GROUP BY gen_subsumer_0.`d_year`, gen_subsumer_0.`i_brand`, gen_subsumer_0.`i_brand_id`
@@ -234,7 +211,7 @@ object TestTPCDS_1_4_Batch {
| FROM
| date_dim dt
| INNER JOIN store_sales ON (dt.`d_date_sk` = store_sales.`ss_sold_date_sk`)
- | INNER JOIN item ON (store_sales.`ss_item_sk` = item.`i_item_sk`)
+ | INNER JOIN item ON (store_sales.`ss_item_sk` = item.`i_item_sk`)
| GROUP BY dt.`d_date`, dt.`d_moy`, dt.`d_year`, item.`i_brand`, item.`i_brand_id`, item.`i_item_id`, item.`i_item_desc`, item.`i_manufact_id`, substring(item.`i_item_desc`, 1, 30), item.`i_category`, item.`i_category_id`, item.`i_class`, item.`i_class_id`, item.`i_current_price`, item.`i_manager_id`, item.`i_item_sk`, store_sales.`ss_store_sk`) gen_subsumer_0
| WHERE
| (gen_subsumer_0.`d_year` IN (2000, 2001, 2002, 2003))
@@ -242,8 +219,8 @@ object TestTPCDS_1_4_Batch {
| LEFT SEMI JOIN (SELECT customer.`c_customer_sk`, sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#271 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#274 as decimal(12,2)))), DecimalType(18,2)))`
| FROM
| store_sales
- | INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`)
- | GROUP BY customer.`c_customer_sk`) gen_subquery_1 ON (CAST(gen_subquery_1.`sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#271 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#274 as decimal(12,2)))), DecimalType(18,2)))` AS DECIMAL(38,8)) > (0.500000BD * CAST((SELECT max(gen_scalar_subquery_0_0.`csales`) AS `tpcds_cmax` FROM (SELECT sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales` FROM store_sales INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`) INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`) GROUP BY customer.`c_customer_sk`) gen_scalar_subquery_0_0 ) AS DECIMAL(32,6)))) AND (catalog_sales.`cs_bill_customer_sk` = gen_subquery_1.`c_customer_sk`)
+ | INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`)
+ | GROUP BY customer.`c_customer_sk`) gen_subquery_1 ON (CAST(gen_subquery_1.`sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#271 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#274 as decimal(12,2)))), DecimalType(18,2)))` AS DECIMAL(38,8)) > (0.500000BD * CAST((SELECT max(gen_expression_0_0.`csales`) AS `tpcds_cmax` FROM (SELECT sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales` FROM store_sales INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`) INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`) GROUP BY customer.`c_customer_sk`) gen_expression_0_0 ) AS DECIMAL(32,6)))) AND (catalog_sales.`cs_bill_customer_sk` = gen_subquery_1.`c_customer_sk`)
| INNER JOIN date_dim ON (date_dim.`d_year` = 2000) AND (date_dim.`d_moy` = 2) AND (catalog_sales.`cs_sold_date_sk` = date_dim.`d_date_sk`)
| UNION ALL
| SELECT (CAST(CAST(web_sales.`ws_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(web_sales.`ws_list_price` AS DECIMAL(12,2))) AS `sales`
@@ -255,7 +232,7 @@ object TestTPCDS_1_4_Batch {
| FROM
| date_dim dt
| INNER JOIN store_sales ON (dt.`d_date_sk` = store_sales.`ss_sold_date_sk`)
- | INNER JOIN item ON (store_sales.`ss_item_sk` = item.`i_item_sk`)
+ | INNER JOIN item ON (store_sales.`ss_item_sk` = item.`i_item_sk`)
| GROUP BY dt.`d_date`, dt.`d_moy`, dt.`d_year`, item.`i_brand`, item.`i_brand_id`, item.`i_item_id`, item.`i_item_desc`, item.`i_manufact_id`, substring(item.`i_item_desc`, 1, 30), item.`i_category`, item.`i_category_id`, item.`i_class`, item.`i_class_id`, item.`i_current_price`, item.`i_manager_id`, item.`i_item_sk`, store_sales.`ss_store_sk`) gen_subsumer_1
| WHERE
| (gen_subsumer_1.`d_year` IN (2000, 2001, 2002, 2003))
@@ -263,8 +240,8 @@ object TestTPCDS_1_4_Batch {
| LEFT SEMI JOIN (SELECT customer.`c_customer_sk`, sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#271 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#274 as decimal(12,2)))), DecimalType(18,2)))`
| FROM
| store_sales
- | INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`)
- | GROUP BY customer.`c_customer_sk`) gen_subquery_3 ON (CAST(gen_subquery_3.`sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#271 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#274 as decimal(12,2)))), DecimalType(18,2)))` AS DECIMAL(38,8)) > (0.500000BD * CAST((SELECT max(gen_scalar_subquery_1_0.`csales`) AS `tpcds_cmax` FROM (SELECT sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales` FROM store_sales INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`) INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`) GROUP BY customer.`c_customer_sk`) gen_scalar_subquery_1_0 ) AS DECIMAL(32,6)))) AND (web_sales.`ws_bill_customer_sk` = gen_subquery_3.`c_customer_sk`)
+ | INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`)
+ | GROUP BY customer.`c_customer_sk`) gen_subquery_3 ON (CAST(gen_subquery_3.`sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#271 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#274 as decimal(12,2)))), DecimalType(18,2)))` AS DECIMAL(38,8)) > (0.500000BD * CAST((SELECT max(gen_expression_1_0.`csales`) AS `tpcds_cmax` FROM (SELECT sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales` FROM store_sales INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`) INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`) GROUP BY customer.`c_customer_sk`) gen_expression_1_0 ) AS DECIMAL(32,6)))) AND (web_sales.`ws_bill_customer_sk` = gen_subquery_3.`c_customer_sk`)
| INNER JOIN date_dim ON (date_dim.`d_year` = 2000) AND (date_dim.`d_moy` = 2) AND (web_sales.`ws_sold_date_sk` = date_dim.`d_date_sk`)) gen_subquery_4
|LIMIT 100
""".stripMargin.trim),
@@ -365,10 +342,7 @@ object TestTPCDS_1_4_Batch {
| limit 100
""".stripMargin.trim,
"""
- |
- |
- |
- """.stripMargin.trim),
+ """.stripMargin.trim),
// q55
("case_8",
"""
@@ -401,12 +375,12 @@ object TestTPCDS_1_4_Batch {
"""
|SELECT gen_subsumer_0.`i_brand_id` AS `brand_id`, gen_subsumer_0.`i_brand` AS `brand`, sum(gen_subsumer_0.`sum_agg`) AS `ext_price`
|FROM
- | (SELECT `d_date`, `d_moy`, `d_year`, `i_brand`, `i_brand_id`, `i_item_id`, `i_item_desc`, substring(`i_item_desc`, 1, 30) AS `itemdesc`, `i_category`, `i_class`, `i_manager_id`, `i_current_price`, `i_item_sk`, `ss_store_sk`, sum(`ss_ext_sales_price`) AS `sum_agg`, sum((CAST(CAST(`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(`ss_list_price` AS DECIMAL(12,2)))) AS `sales`, count(1) AS `number_sales`
+ | (SELECT dt.`d_date`, dt.`d_moy`, dt.`d_year`, item.`i_brand`, item.`i_brand_id`, item.`i_item_id`, item.`i_item_desc`, substring(item.`i_item_desc`, 1, 30) AS `itemdesc`, item.`i_category`, item.`i_class`, item.`i_manager_id`, item.`i_current_price`, item.`i_item_sk`, store_sales.`ss_store_sk`, sum(store_sales.`ss_ext_sales_price`) AS `sum_agg`, sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_list_price` AS DECIMAL(12,2)))) AS `sales`, count(1) AS `number_sales`
| FROM
| date_dim dt
- | INNER JOIN store_sales ON (`d_date_sk` = `ss_sold_date_sk`)
- | INNER JOIN item ON (`ss_item_sk` = `i_item_sk`)
- | GROUP BY `d_date`, `d_moy`, `d_year`, `i_brand`, `i_brand_id`, `i_item_id`, `i_item_desc`, substring(`i_item_desc`, 1, 30), `i_category`, `i_category_id`, `i_class`, `i_class_id`, `i_current_price`, `i_manager_id`, `i_item_sk`, `ss_store_sk`) gen_subsumer_0
+ | INNER JOIN store_sales ON (dt.`d_date_sk` = store_sales.`ss_sold_date_sk`)
+ | INNER JOIN item ON (store_sales.`ss_item_sk` = item.`i_item_sk`)
+ | GROUP BY dt.`d_date`, dt.`d_moy`, dt.`d_year`, item.`i_brand`, item.`i_brand_id`, item.`i_item_id`, item.`i_item_desc`, substring(item.`i_item_desc`, 1, 30), item.`i_category`, item.`i_category_id`, item.`i_class`, item.`i_class_id`, item.`i_current_price`, item.`i_manager_id`, item.`i_item_sk`, store_sales.`ss_store_sk`) gen_subsumer_0
|WHERE
| (gen_subsumer_0.`d_moy` = 11) AND (gen_subsumer_0.`d_year` = 1999) AND (gen_subsumer_0.`i_manager_id` = 28)
|GROUP BY gen_subsumer_0.`i_brand`, gen_subsumer_0.`i_brand_id`
@@ -448,14 +422,28 @@ object TestTPCDS_1_4_Batch {
| i_category, i_class, i_item_id, i_item_desc, revenueratio
""".stripMargin.trim,
"""
- |
- |
- |
+ |SELECT gen_subquery_1.`i_item_desc`, gen_subquery_1.`i_category`, gen_subquery_1.`i_class`, gen_subquery_1.`i_current_price`, gen_subquery_1.`itemrevenue`, ((gen_subquery_1.`_w0` * 100.00BD) / CAST(gen_subquery_1.`_we0` AS DECIMAL(28,2))) AS `revenueratio`
+ |FROM
+ | (SELECT gen_subquery_0.`i_item_desc`, gen_subquery_0.`i_category`, gen_subquery_0.`i_class`, gen_subquery_0.`i_current_price`, gen_subquery_0.`itemrevenue`, gen_subquery_0.`_w0`, gen_subquery_0.`_w1`, gen_subquery_0.`i_item_id`, sum(gen_subquery_0.`_w1`) OVER (PARTITION BY gen_subquery_0.`i_class` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `_we0`
+ | FROM
+ | (SELECT gen_subsumer_0.`i_item_desc`, gen_subsumer_0.`i_category`, gen_subsumer_0.`i_class`, gen_subsumer_0.`i_current_price`, sum(gen_subsumer_0.`sum_agg`) AS `itemrevenue`, sum(gen_subsumer_0.`sum_agg`) AS `_w0`, sum(gen_subsumer_0.`sum_agg`) AS `_w1`, gen_subsumer_0.`i_item_id`
+ | FROM
+ | (SELECT dt.`d_date`, dt.`d_moy`, dt.`d_year`, item.`i_brand`, item.`i_brand_id`, item.`i_item_id`, item.`i_item_desc`, substring(item.`i_item_desc`, 1, 30) AS `itemdesc`, item.`i_category`, item.`i_class`, item.`i_manager_id`, item.`i_current_price`, item.`i_item_sk`, store_sales.`ss_store_sk`, sum(store_sales.`ss_ext_sales_price`) AS `sum_agg`, sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_list_price` AS DECIMAL(12,2)))) AS `sales`, count(1) AS `number_sales`
+ | FROM
+ | date_dim dt
+ | INNER JOIN store_sales ON (dt.`d_date_sk` = store_sales.`ss_sold_date_sk`)
+ | INNER JOIN item ON (store_sales.`ss_item_sk` = item.`i_item_sk`)
+ | GROUP BY dt.`d_date`, dt.`d_moy`, dt.`d_year`, item.`i_brand`, item.`i_brand_id`, item.`i_item_id`, item.`i_item_desc`, substring(item.`i_item_desc`, 1, 30), item.`i_category`, item.`i_category_id`, item.`i_class`, item.`i_class_id`, item.`i_current_price`, item.`i_manager_id`, item.`i_item_sk`, store_sales.`ss_store_sk`) gen_subsumer_0
+ | WHERE
+ | (gen_subsumer_0.`i_category` IN ('Sports', 'Books', 'Home')) AND (gen_subsumer_0.`d_date` >= DATE '1999-02-22') AND (gen_subsumer_0.`d_date` <= DATE '1999-03-24')
+ | GROUP BY gen_subsumer_0.`i_item_id`, gen_subsumer_0.`i_item_desc`, gen_subsumer_0.`i_category`, gen_subsumer_0.`i_class`, gen_subsumer_0.`i_current_price`) gen_subquery_0 ) gen_subquery_1
+ |ORDER BY gen_subquery_1.`i_category` ASC NULLS FIRST, gen_subquery_1.`i_class` ASC NULLS FIRST, gen_subquery_1.`i_item_id` ASC NULLS FIRST, gen_subquery_1.`i_item_desc` ASC NULLS FIRST, `revenueratio` ASC NULLS FIRST
""".stripMargin.trim),
- // q76
+ // q76
+ // this case requires a rule of PushAggregateThroughUnion for rewrite to work, which is not implemented for now
("case_10",
"""
- |SELECT dt.d_date, dt.d_moy, dt.d_year, item.i_brand, item.i_brand_id, item.i_item_id, item.i_item_desc,
+ |SELECT dt.d_date, dt.d_moy, dt.d_year, dt.d_qoy, item.i_brand, item.i_brand_id, item.i_item_id, item.i_item_desc,
| substr(item.i_item_desc, 1, 30) itemdesc, item.i_category, item.i_class,
| item.i_manager_id, item.i_current_price, item.i_item_sk, store_sales.ss_store_sk,
| SUM(store_sales.ss_ext_sales_price) sum_agg,
@@ -463,7 +451,7 @@ object TestTPCDS_1_4_Batch {
|FROM date_dim dt, store_sales, item
|WHERE dt.d_date_sk = store_sales.ss_sold_date_sk
| AND store_sales.ss_item_sk = item.i_item_sk
- |GROUP BY dt.d_date, dt.d_moy, dt.d_year, item.i_brand, item.i_brand_id, item.i_item_id, item.i_item_desc,
+ |GROUP BY dt.d_date, dt.d_moy, dt.d_qoy, dt.d_year, item.i_brand, item.i_brand_id, item.i_item_id, item.i_item_desc,
| substr(item.i_item_desc, 1, 30), item.i_category, item.i_category_id,
| item.i_class, item.i_class_id, item.i_current_price, item.i_manager_id,
| item.i_item_sk, store_sales.ss_store_sk
@@ -501,15 +489,12 @@ object TestTPCDS_1_4_Batch {
| limit 100
""".stripMargin.trim,
"""
- |
- |
- |
- """.stripMargin.trim),
+ """.stripMargin.trim),
// the following four cases involve a MV of catalog_sales, item, date_dim
// q20
("case_11",
"""
- |SELECT cs_ship_addr_sk , d_date, d_year, d_qoy, d_moy, i_category,i_item_sk, i_item_id,
+ |SELECT cs_ship_addr_sk , d_date, d_year, d_qoy, d_moy, i_category, cs_ship_addr_sk,i_item_sk, i_item_id,
| i_item_desc, i_class, i_current_price, i_brand_id, i_class_id, i_category_id, i_manufact_id,
| SUM(cs_ext_sales_price) sales_amt,
| SUM(cs_quantity*cs_list_price) sales,
@@ -546,19 +531,19 @@ object TestTPCDS_1_4_Batch {
| FROM
| (SELECT gen_subsumer_0.`i_item_desc`, gen_subsumer_0.`i_category`, gen_subsumer_0.`i_class`, gen_subsumer_0.`i_current_price`, sum(gen_subsumer_0.`sales_amt`) AS `itemrevenue`, sum(gen_subsumer_0.`sales_amt`) AS `_w0`, sum(gen_subsumer_0.`sales_amt`) AS `_w1`, gen_subsumer_0.`i_item_id`
| FROM
- | (SELECT `cs_ship_addr_sk`, `d_date`, `d_year`, `d_qoy`, `d_moy`, `i_category`, `cs_ship_addr_sk`, `i_item_sk`, `i_item_id`, `i_item_desc`, `i_class`, `i_current_price`, `i_brand_id`, `i_class_id`, `i_category_id`, `i_manufact_id`, sum(`cs_ext_sales_price`) AS `sales_amt`, sum((CAST(CAST(`cs_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(`cs_list_price` AS DECIMAL(12,2)))) AS `sales`, sum(`cs_ext_discount_amt`) AS `excess discount amount`, count(1) AS `number_sales`
+ | (SELECT catalog_sales.`cs_ship_addr_sk`, date_dim.`d_date`, date_dim.`d_year`, date_dim.`d_qoy`, date_dim.`d_moy`, item.`i_category`, catalog_sales.`cs_ship_addr_sk`, item.`i_item_sk`, item.`i_item_id`, item.`i_item_desc`, item.`i_class`, item.`i_current_price`, item.`i_brand_id`, item.`i_class_id`, item.`i_category_id`, item.`i_manufact_id`, sum(catalog_sales.`cs_ext_sales_price`) AS `sales_amt`, sum((CAST(CAST(catalog_sales.`cs_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(catalog_sales.`cs_list_price` AS DECIMAL(12,2)))) AS `sales`, sum(catalog_sales.`cs_ext_discount_amt`) AS `excess discount amount`, count(1) AS `number_sales`
| FROM
| catalog_sales
- | INNER JOIN item ON (`cs_item_sk` = `i_item_sk`)
- | INNER JOIN date_dim ON (`cs_sold_date_sk` = `d_date_sk`)
- | GROUP BY `i_brand_id`, `i_class_id`, `i_category_id`, `i_item_id`, `i_item_desc`, `i_category`, `i_class`, `i_current_price`, `i_manufact_id`, `d_date`, `d_moy`, `d_qoy`, `d_year`, `cs_ship_addr_sk`, `i_item_sk`) gen_subsumer_0
+ | INNER JOIN item ON (catalog_sales.`cs_item_sk` = item.`i_item_sk`)
+ | INNER JOIN date_dim ON (catalog_sales.`cs_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY item.`i_brand_id`, item.`i_class_id`, item.`i_category_id`, item.`i_item_id`, item.`i_item_desc`, item.`i_category`, item.`i_class`, item.`i_current_price`, item.`i_manufact_id`, date_dim.`d_date`, date_dim.`d_moy`, date_dim.`d_qoy`, date_dim.`d_year`, catalog_sales.`cs_ship_addr_sk`, item.`i_item_sk`) gen_subsumer_0
| WHERE
| (gen_subsumer_0.`i_category` IN ('Sports', 'Books', 'Home')) AND (gen_subsumer_0.`d_date` >= DATE '1999-02-22') AND (gen_subsumer_0.`d_date` <= DATE '1999-03-24')
| GROUP BY gen_subsumer_0.`i_item_id`, gen_subsumer_0.`i_item_desc`, gen_subsumer_0.`i_category`, gen_subsumer_0.`i_class`, gen_subsumer_0.`i_current_price`) gen_subquery_0 ) gen_subquery_1
|ORDER BY gen_subquery_1.`i_category` ASC NULLS FIRST, gen_subquery_1.`i_class` ASC NULLS FIRST, gen_subquery_1.`i_item_id` ASC NULLS FIRST, gen_subquery_1.`i_item_desc` ASC NULLS FIRST, `revenueratio` ASC NULLS FIRST
|LIMIT 100
""".stripMargin.trim),
- // q32
+ // q32 - no rewrite
("case_12",
"""
|SELECT cs_ship_addr_sk , d_date, d_year, d_qoy, d_moy, i_category, cs_ship_addr_sk,i_item_sk, i_item_id,
@@ -591,18 +576,15 @@ object TestTPCDS_1_4_Batch {
|limit 100
""".stripMargin.trim,
"""
- |
- |
- |
- """.stripMargin.trim),
- // q58 debug
+ """.stripMargin.trim),
+ // q58 debug - no rewrite
("case_13",
"""
|SELECT cs_ship_addr_sk , d_date, d_year, d_qoy, d_moy, i_category, cs_ext_sales_price, cs_ship_addr_sk, i_item_sk, i_item_id,
| i_item_desc, i_class, i_current_price, i_brand_id, i_class_id, i_category_id, i_manufact_id,
| SUM(cs_ext_sales_price) sales_amt,
| SUM(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over
- | (partition by i_class) as revenueratio
+ | (partition by i_class) as revenueratio,
| SUM(cs_quantity*cs_list_price) sales,
| SUM(cs_ext_discount_amt) as `excess discount amount`,
| count(*) number_sales
@@ -610,7 +592,7 @@ object TestTPCDS_1_4_Batch {
|WHERE cs_item_sk = i_item_sk
| AND cs_sold_date_sk = d_date_sk
|GROUP BY i_brand_id, i_class_id, i_category_id, i_item_id, i_item_desc, i_category, i_class,
- | i_current_price, i_manufact_id, d_date, d_moy, d_qoy, d_year, cs_ship_addr_sk, i_item_sk
+ | i_current_price, i_manufact_id, d_date, d_moy, d_qoy, d_year, cs_ext_sales_price, cs_ship_addr_sk, i_item_sk
""".stripMargin.trim,
"""
| with ss_items as
@@ -668,11 +650,8 @@ object TestTPCDS_1_4_Batch {
| limit 100
""".stripMargin.trim,
"""
- |
- |
- |
- """.stripMargin.trim),
- // q76
+ """.stripMargin.trim),
+ // q76 - as case 10, require the rule PushAggregateThroughUnion
("case_14",
"""
|SELECT cs_ship_addr_sk , d_date, d_year, d_qoy, d_moy, i_category, cs_ext_sales_price, cs_ship_addr_sk, i_item_sk, i_item_id,
@@ -685,7 +664,8 @@ object TestTPCDS_1_4_Batch {
|WHERE cs_item_sk = i_item_sk
| AND cs_sold_date_sk = d_date_sk
|GROUP BY i_brand_id, i_class_id, i_category_id, i_item_id, i_item_desc, i_category, i_class,
- | i_current_price, i_manufact_id, d_date, d_moy, d_qoy, d_year, cs_ship_addr_sk, i_item_sk
+ | i_current_price, i_manufact_id, d_date, d_moy, d_qoy, d_year, cs_ship_addr_sk, i_item_sk,
+ | cs_ext_sales_price
""".stripMargin.trim,
"""
| SELECT
@@ -720,10 +700,7 @@ object TestTPCDS_1_4_Batch {
| limit 100
""".stripMargin.trim,
"""
- |
- |
- |
- """.stripMargin.trim),
+ """.stripMargin.trim),
// the following two cases involve a MV of store_sales and customer
// q23a
("case_15",
@@ -777,10 +754,45 @@ object TestTPCDS_1_4_Batch {
| limit 100
""".stripMargin.trim,
"""
- |
- |
- |
- """.stripMargin.trim),
+ |SELECT sum(gen_subquery_4.`sales`) AS `sum(sales)`
+ |FROM
+ | (SELECT (CAST(CAST(catalog_sales.`cs_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(catalog_sales.`cs_list_price` AS DECIMAL(12,2))) AS `sales`
+ | FROM
+ | catalog_sales
+ | LEFT SEMI JOIN (SELECT item.`i_item_sk` AS `item_sk`, count(1) AS `count(1)`
+ | FROM
+ | store_sales
+ | INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
+ | INNER JOIN item ON (store_sales.`ss_item_sk` = item.`i_item_sk`)
+ | GROUP BY substring(item.`i_item_desc`, 1, 30), item.`i_item_sk`, date_dim.`d_date`) gen_subquery_0 ON (gen_subquery_0.`count(1)` > 4L) AND (catalog_sales.`cs_item_sk` = gen_subquery_0.`item_sk`)
+ | LEFT SEMI JOIN (SELECT customer.`c_customer_sk`, sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales`
+ | FROM
+ | customer
+ | INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
+ | GROUP BY customer.`c_customer_sk`) gen_subquery_1 ON (catalog_sales.`cs_bill_customer_sk` = gen_subquery_1.`c_customer_sk`)
+ | INNER JOIN date_dim ON (date_dim.`d_year` = 2000) AND (date_dim.`d_moy` = 2) AND (catalog_sales.`cs_sold_date_sk` = date_dim.`d_date_sk`)
+ | WHERE
+ | (CAST(`sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#219 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#222 as decimal(12,2)))), DecimalType(18,2)))` AS DECIMAL(38,8)) > (0.500000BD * CAST((SELECT max(gen_expression_0_0.`csales`) AS `tpcds_cmax` FROM (SELECT sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales` FROM store_sales INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`) INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`) GROUP BY customer.`c_customer_sk`) gen_expression_0_0 ) AS DECIMAL(32,6))))
+ | UNION ALL
+ | SELECT (CAST(CAST(web_sales.`ws_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(web_sales.`ws_list_price` AS DECIMAL(12,2))) AS `sales`
+ | FROM
+ | web_sales
+ | LEFT SEMI JOIN (SELECT item.`i_item_sk` AS `item_sk`, count(1) AS `count(1)`
+ | FROM
+ | store_sales
+ | INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
+ | INNER JOIN item ON (store_sales.`ss_item_sk` = item.`i_item_sk`)
+ | GROUP BY substring(item.`i_item_desc`, 1, 30), item.`i_item_sk`, date_dim.`d_date`) gen_subquery_2 ON (gen_subquery_2.`count(1)` > 4L) AND (web_sales.`ws_item_sk` = gen_subquery_2.`item_sk`)
+ | LEFT SEMI JOIN (SELECT customer.`c_customer_sk`, sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales`
+ | FROM
+ | customer
+ | INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
+ | GROUP BY customer.`c_customer_sk`) gen_subquery_3 ON (web_sales.`ws_bill_customer_sk` = gen_subquery_3.`c_customer_sk`)
+ | INNER JOIN date_dim ON (date_dim.`d_year` = 2000) AND (date_dim.`d_moy` = 2) AND (web_sales.`ws_sold_date_sk` = date_dim.`d_date_sk`)
+ | WHERE
+ | (CAST(`sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#219 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#222 as decimal(12,2)))), DecimalType(18,2)))` AS DECIMAL(38,8)) > (0.500000BD * CAST((SELECT max(gen_expression_1_0.`csales`) AS `tpcds_cmax` FROM (SELECT sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales` FROM store_sales INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`) INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`) GROUP BY customer.`c_customer_sk`) gen_expression_1_0 ) AS DECIMAL(32,6))))) gen_subquery_4
+ |LIMIT 100
+ """.stripMargin.trim),
// q23b
("case_16",
"""
@@ -791,7 +803,6 @@ object TestTPCDS_1_4_Batch {
| GROUP BY c_customer_sk
""".stripMargin.trim,
"""
- |
| with frequent_ss_items as
| (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt
| from store_sales, date_dim, item
@@ -851,17 +862,17 @@ object TestTPCDS_1_4_Batch {
| FROM
| store_sales
| INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
- | INNER JOIN item ON (store_sales.`ss_item_sk` = item.`i_item_sk`)
+ | INNER JOIN item ON (store_sales.`ss_item_sk` = item.`i_item_sk`)
| GROUP BY substring(item.`i_item_desc`, 1, 30), item.`i_item_sk`, date_dim.`d_date`) gen_subquery_0 ON (gen_subquery_0.`count(1)` > 4L) AND (catalog_sales.`cs_item_sk` = gen_subquery_0.`item_sk`)
| LEFT SEMI JOIN (SELECT customer.`c_customer_sk`, sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales`
| FROM
- | customer
+ | customer
| INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
| GROUP BY customer.`c_customer_sk`) gen_subquery_1 ON (catalog_sales.`cs_bill_customer_sk` = gen_subquery_1.`c_customer_sk`)
| INNER JOIN customer ON (catalog_sales.`cs_bill_customer_sk` = customer.`c_customer_sk`)
| INNER JOIN date_dim ON (date_dim.`d_year` = 2000) AND (date_dim.`d_moy` = 2) AND (catalog_sales.`cs_sold_date_sk` = date_dim.`d_date_sk`)
| WHERE
- | (CAST(`sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#219 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#222 as decimal(12,2)))), DecimalType(18,2)))` AS DECIMAL(38,8)) > (0.500000BD * CAST((SELECT max(gen_expression_0_0.`csales`) AS `tpcds_cmax` FROM (SELECT sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales` FROM store_sales INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`) INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`) GROUP BY customer.`c_customer_sk`) gen_expression_0_0 ) AS DECIMAL(32,6))))) gen_subquery_2
+ | (CAST(`sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#219 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#222 as decimal(12,2)))), DecimalType(18,2)))` AS DECIMAL(38,8)) > (0.500000BD * CAST((SELECT max(gen_expression_0_0.`csales`) AS `tpcds_cmax` FROM (SELECT sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales` FROM store_sales INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`) INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`) GROUP BY customer.`c_customer_sk`) gen_expression_0_0 ) AS DECIMAL(32,6))))) gen_subquery_2
| GROUP BY gen_subquery_2.`c_last_name`, gen_subquery_2.`c_first_name`
| UNION ALL
| SELECT gen_subquery_5.`c_last_name`, gen_subquery_5.`c_first_name`, sum((CAST(CAST(gen_subquery_5.`ws_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(gen_subquery_5.`ws_list_price` AS DECIMAL(12,2)))) AS `sales`
@@ -873,17 +884,17 @@ object TestTPCDS_1_4_Batch {
| FROM
| store_sales
| INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
- | INNER JOIN item ON (store_sales.`ss_item_sk` = item.`i_item_sk`)
+ | INNER JOIN item ON (store_sales.`ss_item_sk` = item.`i_item_sk`)
| GROUP BY substring(item.`i_item_desc`, 1, 30), item.`i_item_sk`, date_dim.`d_date`) gen_subquery_3 ON (gen_subquery_3.`count(1)` > 4L) AND (web_sales.`ws_item_sk` = gen_subquery_3.`item_sk`)
| LEFT SEMI JOIN (SELECT customer.`c_customer_sk`, sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales`
| FROM
- | customer
+ | customer
| INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
| GROUP BY customer.`c_customer_sk`) gen_subquery_4 ON (web_sales.`ws_bill_customer_sk` = gen_subquery_4.`c_customer_sk`)
| INNER JOIN customer ON (web_sales.`ws_bill_customer_sk` = customer.`c_customer_sk`)
| INNER JOIN date_dim ON (date_dim.`d_year` = 2000) AND (date_dim.`d_moy` = 2) AND (web_sales.`ws_sold_date_sk` = date_dim.`d_date_sk`)
| WHERE
- | (CAST(`sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#219 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#222 as decimal(12,2)))), DecimalType(18,2)))` AS DECIMAL(38,8)) > (0.500000BD * CAST((SELECT max(gen_expression_1_0.`csales`) AS `tpcds_cmax` FROM (SELECT sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales` FROM store_sales INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`) INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`) GROUP BY customer.`c_customer_sk`) gen_expression_1_0 ) AS DECIMAL(32,6))))) gen_subquery_5
+ | (CAST(`sum(CheckOverflow((promote_precision(cast(cast(ss_quantity#219 as decimal(10,0)) as decimal(12,2))) * promote_precision(cast(ss_sales_price#222 as decimal(12,2)))), DecimalType(18,2)))` AS DECIMAL(38,8)) > (0.500000BD * CAST((SELECT max(gen_expression_1_0.`csales`) AS `tpcds_cmax` FROM (SELECT sum((CAST(CAST(store_sales.`ss_quantity` AS DECIMAL(10,0)) AS DECIMAL(12,2)) * CAST(store_sales.`ss_sales_price` AS DECIMAL(12,2)))) AS `csales` FROM store_sales INNER JOIN customer ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`) INNER JOIN date_dim ON (date_dim.`d_year` IN (2000, 2001, 2002, 2003)) AND (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`) GROUP BY customer.`c_customer_sk`) gen_expression_1_0 ) AS DECIMAL(32,6))))) gen_subquery_5
| GROUP BY gen_subquery_5.`c_last_name`, gen_subquery_5.`c_first_name`) gen_subquery_6
|ORDER BY gen_subquery_6.`c_last_name` ASC NULLS FIRST, gen_subquery_6.`c_first_name` ASC NULLS FIRST, gen_subquery_6.`sales` ASC NULLS FIRST
|LIMIT 100
@@ -1026,9 +1037,187 @@ object TestTPCDS_1_4_Batch {
| LIMIT 100
""".stripMargin.trim,
"""
- |
- |
- |
+ |SELECT gen_subquery_1.`customer_id`, gen_subquery_1.`customer_first_name`, gen_subquery_1.`customer_last_name`, gen_subquery_1.`customer_preferred_cust_flag`, gen_subquery_1.`customer_birth_country`, gen_subquery_1.`customer_login`, gen_subquery_1.`customer_email_address`
+ |FROM
+ | (SELECT gen_subsumer_0.`customer_id` AS `customer_id`, sum(gen_subsumer_0.`year_total`) AS `year_total`
+ | FROM
+ | (SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, date_dim.`d_year` AS `dyear`, date_dim.`d_date` AS `ddate`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(store_sales.`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
+ | FROM
+ | customer
+ | INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
+ | INNER JOIN date_dim ON (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`, date_dim.`d_date`) gen_subsumer_0
+ | WHERE
+ | (gen_subsumer_0.`dyear` = 2001)
+ | GROUP BY gen_subsumer_0.`customer_id`, gen_subsumer_0.`customer_first_name`, gen_subsumer_0.`customer_last_name`, gen_subsumer_0.`customer_preferred_cust_flag`, gen_subsumer_0.`customer_birth_country`, gen_subsumer_0.`customer_login`, gen_subsumer_0.`customer_email_address`, gen_subsumer_0.`dyear`
+ | HAVING (sum(gen_subsumer_0.`year_total`) > 0E-13BD)
+ | UNION ALL
+ | SELECT customer.`c_customer_id` AS `customer_id`, sum((CAST((((CAST(catalog_sales.`cs_ext_list_price` AS DECIMAL(8,2)) - CAST(catalog_sales.`cs_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(catalog_sales.`cs_ext_discount_amt` AS DECIMAL(8,2))) + CAST(catalog_sales.`cs_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`
+ | FROM
+ | customer
+ | INNER JOIN catalog_sales ON (customer.`c_customer_sk` = catalog_sales.`cs_bill_customer_sk`)
+ | INNER JOIN date_dim ON (catalog_sales.`cs_sold_date_sk` = date_dim.`d_date_sk`)
+ | WHERE
+ | false
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`
+ | HAVING (`year_total` > 0E-13BD)
+ | UNION ALL
+ | SELECT customer.`c_customer_id` AS `customer_id`, sum((CAST((((CAST(web_sales.`ws_ext_list_price` AS DECIMAL(8,2)) - CAST(web_sales.`ws_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(web_sales.`ws_ext_discount_amt` AS DECIMAL(8,2))) + CAST(web_sales.`ws_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`
+ | FROM
+ | customer
+ | INNER JOIN web_sales ON (customer.`c_customer_sk` = web_sales.`ws_bill_customer_sk`)
+ | INNER JOIN date_dim ON (web_sales.`ws_sold_date_sk` = date_dim.`d_date_sk`)
+ | WHERE
+ | false
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`
+ | HAVING (`year_total` > 0E-13BD)) gen_subquery_0
+ | INNER JOIN (SELECT gen_subsumer_1.`customer_id` AS `customer_id`, gen_subsumer_1.`customer_first_name` AS `customer_first_name`, gen_subsumer_1.`customer_last_name` AS `customer_last_name`, gen_subsumer_1.`customer_preferred_cust_flag` AS `customer_preferred_cust_flag`, gen_subsumer_1.`customer_birth_country` AS `customer_birth_country`, gen_subsumer_1.`customer_login` AS `customer_login`, gen_subsumer_1.`customer_email_address` AS `customer_email_address`, sum(gen_subsumer_1.`year_total`) AS `year_total`
+ | FROM
+ | (SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, date_dim.`d_year` AS `dyear`, date_dim.`d_date` AS `ddate`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(store_sales.`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
+ | FROM
+ | customer
+ | INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
+ | INNER JOIN date_dim ON (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`, date_dim.`d_date`) gen_subsumer_1
+ | WHERE
+ | (gen_subsumer_1.`dyear` = 2002)
+ | GROUP BY gen_subsumer_1.`customer_id`, gen_subsumer_1.`customer_first_name`, gen_subsumer_1.`customer_last_name`, gen_subsumer_1.`customer_preferred_cust_flag`, gen_subsumer_1.`customer_birth_country`, gen_subsumer_1.`customer_login`, gen_subsumer_1.`customer_email_address`, gen_subsumer_1.`dyear`
+ | UNION ALL
+ | SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, sum((CAST((((CAST(catalog_sales.`cs_ext_list_price` AS DECIMAL(8,2)) - CAST(catalog_sales.`cs_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(catalog_sales.`cs_ext_discount_amt` AS DECIMAL(8,2))) + CAST(catalog_sales.`cs_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`
+ | FROM
+ | customer
+ | INNER JOIN catalog_sales ON (customer.`c_customer_sk` = catalog_sales.`cs_bill_customer_sk`)
+ | INNER JOIN date_dim ON (catalog_sales.`cs_sold_date_sk` = date_dim.`d_date_sk`)
+ | WHERE
+ | false
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`
+ | UNION ALL
+ | SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, sum((CAST((((CAST(web_sales.`ws_ext_list_price` AS DECIMAL(8,2)) - CAST(web_sales.`ws_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(web_sales.`ws_ext_discount_amt` AS DECIMAL(8,2))) + CAST(web_sales.`ws_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`
+ | FROM
+ | customer
+ | INNER JOIN web_sales ON (customer.`c_customer_sk` = web_sales.`ws_bill_customer_sk`)
+ | INNER JOIN date_dim ON (web_sales.`ws_sold_date_sk` = date_dim.`d_date_sk`)
+ | WHERE
+ | false
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`) gen_subquery_1 ON (gen_subquery_1.`customer_id` = gen_subquery_0.`customer_id`)
+ | INNER JOIN (SELECT gen_subsumer_2.`customer_id` AS `customer_id`, sum(gen_subsumer_2.`year_total`) AS `year_total`
+ | FROM
+ | (SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, date_dim.`d_year` AS `dyear`, date_dim.`d_date` AS `ddate`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(store_sales.`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
+ | FROM
+ | customer
+ | INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
+ | INNER JOIN date_dim ON (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`, date_dim.`d_date`) gen_subsumer_2
+ | WHERE
+ | false
+ | GROUP BY gen_subsumer_2.`customer_id`, gen_subsumer_2.`customer_first_name`, gen_subsumer_2.`customer_last_name`, gen_subsumer_2.`customer_preferred_cust_flag`, gen_subsumer_2.`customer_birth_country`, gen_subsumer_2.`customer_login`, gen_subsumer_2.`customer_email_address`, gen_subsumer_2.`dyear`
+ | HAVING (sum(gen_subsumer_2.`year_total`) > 0E-13BD)
+ | UNION ALL
+ | SELECT customer.`c_customer_id` AS `customer_id`, sum((CAST((((CAST(catalog_sales.`cs_ext_list_price` AS DECIMAL(8,2)) - CAST(catalog_sales.`cs_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(catalog_sales.`cs_ext_discount_amt` AS DECIMAL(8,2))) + CAST(catalog_sales.`cs_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`
+ | FROM
+ | customer
+ | INNER JOIN catalog_sales ON (customer.`c_customer_sk` = catalog_sales.`cs_bill_customer_sk`)
+ | INNER JOIN date_dim ON (date_dim.`d_year` = 2001) AND (catalog_sales.`cs_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`
+ | HAVING (`year_total` > 0E-13BD)
+ | UNION ALL
+ | SELECT customer.`c_customer_id` AS `customer_id`, sum((CAST((((CAST(web_sales.`ws_ext_list_price` AS DECIMAL(8,2)) - CAST(web_sales.`ws_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(web_sales.`ws_ext_discount_amt` AS DECIMAL(8,2))) + CAST(web_sales.`ws_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`
+ | FROM
+ | customer
+ | INNER JOIN web_sales ON (customer.`c_customer_sk` = web_sales.`ws_bill_customer_sk`)
+ | INNER JOIN date_dim ON (web_sales.`ws_sold_date_sk` = date_dim.`d_date_sk`)
+ | WHERE
+ | false
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`
+ | HAVING (`year_total` > 0E-13BD)) gen_subquery_2 ON (gen_subquery_0.`customer_id` = gen_subquery_2.`customer_id`)
+ | INNER JOIN (SELECT gen_subsumer_3.`customer_id` AS `customer_id`, sum(gen_subsumer_3.`year_total`) AS `year_total`
+ | FROM
+ | (SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, date_dim.`d_year` AS `dyear`, date_dim.`d_date` AS `ddate`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(store_sales.`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
+ | FROM
+ | customer
+ | INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
+ | INNER JOIN date_dim ON (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`, date_dim.`d_date`) gen_subsumer_3
+ | WHERE
+ | false
+ | GROUP BY gen_subsumer_3.`customer_id`, gen_subsumer_3.`customer_first_name`, gen_subsumer_3.`customer_last_name`, gen_subsumer_3.`customer_preferred_cust_flag`, gen_subsumer_3.`customer_birth_country`, gen_subsumer_3.`customer_login`, gen_subsumer_3.`customer_email_address`, gen_subsumer_3.`dyear`
+ | UNION ALL
+ | SELECT customer.`c_customer_id` AS `customer_id`, sum((CAST((((CAST(catalog_sales.`cs_ext_list_price` AS DECIMAL(8,2)) - CAST(catalog_sales.`cs_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(catalog_sales.`cs_ext_discount_amt` AS DECIMAL(8,2))) + CAST(catalog_sales.`cs_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`
+ | FROM
+ | customer
+ | INNER JOIN catalog_sales ON (customer.`c_customer_sk` = catalog_sales.`cs_bill_customer_sk`)
+ | INNER JOIN date_dim ON (date_dim.`d_year` = 2002) AND (catalog_sales.`cs_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`
+ | UNION ALL
+ | SELECT customer.`c_customer_id` AS `customer_id`, sum((CAST((((CAST(web_sales.`ws_ext_list_price` AS DECIMAL(8,2)) - CAST(web_sales.`ws_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(web_sales.`ws_ext_discount_amt` AS DECIMAL(8,2))) + CAST(web_sales.`ws_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`
+ | FROM
+ | customer
+ | INNER JOIN web_sales ON (customer.`c_customer_sk` = web_sales.`ws_bill_customer_sk`)
+ | INNER JOIN date_dim ON (web_sales.`ws_sold_date_sk` = date_dim.`d_date_sk`)
+ | WHERE
+ | false
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`) gen_subquery_3
+ | INNER JOIN (SELECT gen_subsumer_4.`customer_id` AS `customer_id`, sum(gen_subsumer_4.`year_total`) AS `year_total`
+ | FROM
+ | (SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, date_dim.`d_year` AS `dyear`, date_dim.`d_date` AS `ddate`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(store_sales.`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
+ | FROM
+ | customer
+ | INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
+ | INNER JOIN date_dim ON (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`, date_dim.`d_date`) gen_subsumer_4
+ | WHERE
+ | false
+ | GROUP BY gen_subsumer_4.`customer_id`, gen_subsumer_4.`customer_first_name`, gen_subsumer_4.`customer_last_name`, gen_subsumer_4.`customer_preferred_cust_flag`, gen_subsumer_4.`customer_birth_country`, gen_subsumer_4.`customer_login`, gen_subsumer_4.`customer_email_address`, gen_subsumer_4.`dyear`
+ | HAVING (sum(gen_subsumer_4.`year_total`) > 0E-13BD)
+ | UNION ALL
+ | SELECT customer.`c_customer_id` AS `customer_id`, sum((CAST((((CAST(catalog_sales.`cs_ext_list_price` AS DECIMAL(8,2)) - CAST(catalog_sales.`cs_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(catalog_sales.`cs_ext_discount_amt` AS DECIMAL(8,2))) + CAST(catalog_sales.`cs_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`
+ | FROM
+ | customer
+ | INNER JOIN catalog_sales ON (customer.`c_customer_sk` = catalog_sales.`cs_bill_customer_sk`)
+ | INNER JOIN date_dim ON (catalog_sales.`cs_sold_date_sk` = date_dim.`d_date_sk`)
+ | WHERE
+ | false
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`
+ | HAVING (`year_total` > 0E-13BD)
+ | UNION ALL
+ | SELECT customer.`c_customer_id` AS `customer_id`, sum((CAST((((CAST(web_sales.`ws_ext_list_price` AS DECIMAL(8,2)) - CAST(web_sales.`ws_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(web_sales.`ws_ext_discount_amt` AS DECIMAL(8,2))) + CAST(web_sales.`ws_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`
+ | FROM
+ | customer
+ | INNER JOIN web_sales ON (customer.`c_customer_sk` = web_sales.`ws_bill_customer_sk`)
+ | INNER JOIN date_dim ON (date_dim.`d_year` = 2001) AND (web_sales.`ws_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`
+ | HAVING (`year_total` > 0E-13BD)) gen_subquery_4 ON (gen_subquery_0.`customer_id` = gen_subquery_4.`customer_id`)
+ | INNER JOIN (SELECT gen_subsumer_5.`customer_id` AS `customer_id`, sum(gen_subsumer_5.`year_total`) AS `year_total`
+ | FROM
+ | (SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, date_dim.`d_year` AS `dyear`, date_dim.`d_date` AS `ddate`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(store_sales.`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
+ | FROM
+ | customer
+ | INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
+ | INNER JOIN date_dim ON (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`, date_dim.`d_date`) gen_subsumer_5
+ | WHERE
+ | false
+ | GROUP BY gen_subsumer_5.`customer_id`, gen_subsumer_5.`customer_first_name`, gen_subsumer_5.`customer_last_name`, gen_subsumer_5.`customer_preferred_cust_flag`, gen_subsumer_5.`customer_birth_country`, gen_subsumer_5.`customer_login`, gen_subsumer_5.`customer_email_address`, gen_subsumer_5.`dyear`
+ | UNION ALL
+ | SELECT customer.`c_customer_id` AS `customer_id`, sum((CAST((((CAST(catalog_sales.`cs_ext_list_price` AS DECIMAL(8,2)) - CAST(catalog_sales.`cs_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(catalog_sales.`cs_ext_discount_amt` AS DECIMAL(8,2))) + CAST(catalog_sales.`cs_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`
+ | FROM
+ | customer
+ | INNER JOIN catalog_sales ON (customer.`c_customer_sk` = catalog_sales.`cs_bill_customer_sk`)
+ | INNER JOIN date_dim ON (catalog_sales.`cs_sold_date_sk` = date_dim.`d_date_sk`)
+ | WHERE
+ | false
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`
+ | UNION ALL
+ | SELECT customer.`c_customer_id` AS `customer_id`, sum((CAST((((CAST(web_sales.`ws_ext_list_price` AS DECIMAL(8,2)) - CAST(web_sales.`ws_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(web_sales.`ws_ext_discount_amt` AS DECIMAL(8,2))) + CAST(web_sales.`ws_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`
+ | FROM
+ | customer
+ | INNER JOIN web_sales ON (customer.`c_customer_sk` = web_sales.`ws_bill_customer_sk`)
+ | INNER JOIN date_dim ON (date_dim.`d_year` = 2002) AND (web_sales.`ws_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`) gen_subquery_5
+ |WHERE
+ | (gen_subquery_0.`customer_id` = gen_subquery_3.`customer_id`) AND (CASE WHEN (gen_subquery_2.`year_total` > 0E-13BD) THEN (gen_subquery_3.`year_total` / gen_subquery_2.`year_total`) ELSE CAST(NULL AS DECIMAL(38,23)) END > CASE WHEN (gen_subquery_0.`year_total` > 0E-13BD) THEN (gen_subquery_1.`year_total` / gen_subquery_0.`year_total`) ELSE CAST(NULL AS DECIMAL(38,23)) END) AND (gen_subquery_0.`customer_id` = gen_subquery_5.`customer_id`) AND (CASE WHEN (gen_subquery_2.`year_total` > 0E-13BD) THEN (gen_subquery_3.`year_total` / gen_subquery_2.`year_total`) ELSE CAST(NULL AS DECIMAL(38,23)) END > CASE WHEN (gen_subquery_4.`year_total` > 0E-13BD) THEN (gen_subquery_5.`year_total` / gen_subquery_4.`year_total`) ELSE CAST(NULL AS DECIMAL(38,23)) END)
+ |ORDER BY gen_subquery_1.`customer_id` ASC NULLS FIRST, gen_subquery_1.`customer_first_name` ASC NULLS FIRST, gen_subquery_1.`customer_last_name` ASC NULLS FIRST, gen_subquery_1.`customer_preferred_cust_flag` ASC NULLS FIRST, gen_subquery_1.`customer_birth_country` ASC NULLS FIRST, gen_subquery_1.`customer_login` ASC NULLS FIRST, gen_subquery_1.`customer_email_address` ASC NULLS FIRST
+ |LIMIT 100
""".stripMargin.trim),
//q11
("case_18",
@@ -1128,12 +1317,12 @@ object TestTPCDS_1_4_Batch {
|FROM
| (SELECT gen_subsumer_0.`customer_id` AS `customer_id`, sum(gen_subsumer_0.`year_total1`) AS `year_total`
| FROM
- | (SELECT `c_customer_id` AS `customer_id`, `c_first_name` AS `customer_first_name`, `c_last_name` AS `customer_last_name`, `c_preferred_cust_flag` AS `customer_preferred_cust_flag`, `c_birth_country` AS `customer_birth_country`, `c_login` AS `customer_login`, `c_email_address` AS `customer_email_address`, `d_year` AS `dyear`, `d_date` AS `ddate`, sum((CAST((((CAST(`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
+ | (SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, date_dim.`d_year` AS `dyear`, date_dim.`d_date` AS `ddate`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(store_sales.`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
| FROM
| customer
- | INNER JOIN store_sales ON (`c_customer_sk` = `ss_customer_sk`)
- | INNER JOIN date_dim ON (`ss_sold_date_sk` = `d_date_sk`)
- | GROUP BY `c_customer_id`, `c_first_name`, `c_last_name`, `c_preferred_cust_flag`, `c_birth_country`, `c_login`, `c_email_address`, `d_year`, `d_date`) gen_subsumer_0
+ | INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
+ | INNER JOIN date_dim ON (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`, date_dim.`d_date`) gen_subsumer_0
| WHERE
| (gen_subsumer_0.`dyear` = 2001)
| GROUP BY gen_subsumer_0.`customer_id`, gen_subsumer_0.`customer_first_name`, gen_subsumer_0.`customer_last_name`, gen_subsumer_0.`dyear`, gen_subsumer_0.`customer_preferred_cust_flag`, gen_subsumer_0.`customer_birth_country`, gen_subsumer_0.`customer_login`, gen_subsumer_0.`customer_email_address`
@@ -1150,12 +1339,12 @@ object TestTPCDS_1_4_Batch {
| HAVING (`year_total` > 0.00BD)) gen_subquery_0
| INNER JOIN (SELECT gen_subsumer_1.`customer_id` AS `customer_id`, gen_subsumer_1.`customer_preferred_cust_flag` AS `customer_preferred_cust_flag`, sum(gen_subsumer_1.`year_total1`) AS `year_total`
| FROM
- | (SELECT `c_customer_id` AS `customer_id`, `c_first_name` AS `customer_first_name`, `c_last_name` AS `customer_last_name`, `c_preferred_cust_flag` AS `customer_preferred_cust_flag`, `c_birth_country` AS `customer_birth_country`, `c_login` AS `customer_login`, `c_email_address` AS `customer_email_address`, `d_year` AS `dyear`, `d_date` AS `ddate`, sum((CAST((((CAST(`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
+ | (SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, date_dim.`d_year` AS `dyear`, date_dim.`d_date` AS `ddate`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(store_sales.`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
| FROM
| customer
- | INNER JOIN store_sales ON (`c_customer_sk` = `ss_customer_sk`)
- | INNER JOIN date_dim ON (`ss_sold_date_sk` = `d_date_sk`)
- | GROUP BY `c_customer_id`, `c_first_name`, `c_last_name`, `c_preferred_cust_flag`, `c_birth_country`, `c_login`, `c_email_address`, `d_year`, `d_date`) gen_subsumer_1
+ | INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
+ | INNER JOIN date_dim ON (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`, date_dim.`d_date`) gen_subsumer_1
| WHERE
| (gen_subsumer_1.`dyear` = 2002)
| GROUP BY gen_subsumer_1.`customer_id`, gen_subsumer_1.`customer_first_name`, gen_subsumer_1.`customer_last_name`, gen_subsumer_1.`dyear`, gen_subsumer_1.`customer_preferred_cust_flag`, gen_subsumer_1.`customer_birth_country`, gen_subsumer_1.`customer_login`, gen_subsumer_1.`customer_email_address`
@@ -1170,12 +1359,12 @@ object TestTPCDS_1_4_Batch {
| GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`) gen_subquery_1 ON (gen_subquery_1.`customer_id` = gen_subquery_0.`customer_id`)
| INNER JOIN (SELECT gen_subsumer_2.`customer_id` AS `customer_id`, sum(gen_subsumer_2.`year_total1`) AS `year_total`
| FROM
- | (SELECT `c_customer_id` AS `customer_id`, `c_first_name` AS `customer_first_name`, `c_last_name` AS `customer_last_name`, `c_preferred_cust_flag` AS `customer_preferred_cust_flag`, `c_birth_country` AS `customer_birth_country`, `c_login` AS `customer_login`, `c_email_address` AS `customer_email_address`, `d_year` AS `dyear`, `d_date` AS `ddate`, sum((CAST((((CAST(`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
+ | (SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, date_dim.`d_year` AS `dyear`, date_dim.`d_date` AS `ddate`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(store_sales.`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
| FROM
| customer
- | INNER JOIN store_sales ON (`c_customer_sk` = `ss_customer_sk`)
- | INNER JOIN date_dim ON (`ss_sold_date_sk` = `d_date_sk`)
- | GROUP BY `c_customer_id`, `c_first_name`, `c_last_name`, `c_preferred_cust_flag`, `c_birth_country`, `c_login`, `c_email_address`, `d_year`, `d_date`) gen_subsumer_2
+ | INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
+ | INNER JOIN date_dim ON (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`, date_dim.`d_date`) gen_subsumer_2
| WHERE
| false
| GROUP BY gen_subsumer_2.`customer_id`, gen_subsumer_2.`customer_first_name`, gen_subsumer_2.`customer_last_name`, gen_subsumer_2.`dyear`, gen_subsumer_2.`customer_preferred_cust_flag`, gen_subsumer_2.`customer_birth_country`, gen_subsumer_2.`customer_login`, gen_subsumer_2.`customer_email_address`
@@ -1190,12 +1379,12 @@ object TestTPCDS_1_4_Batch {
| HAVING (`year_total` > 0.00BD)) gen_subquery_2 ON (gen_subquery_0.`customer_id` = gen_subquery_2.`customer_id`)
| INNER JOIN (SELECT gen_subsumer_3.`customer_id` AS `customer_id`, sum(gen_subsumer_3.`year_total1`) AS `year_total`
| FROM
- | (SELECT `c_customer_id` AS `customer_id`, `c_first_name` AS `customer_first_name`, `c_last_name` AS `customer_last_name`, `c_preferred_cust_flag` AS `customer_preferred_cust_flag`, `c_birth_country` AS `customer_birth_country`, `c_login` AS `customer_login`, `c_email_address` AS `customer_email_address`, `d_year` AS `dyear`, `d_date` AS `ddate`, sum((CAST((((CAST(`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
+ | (SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, date_dim.`d_year` AS `dyear`, date_dim.`d_date` AS `ddate`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2)))) AS `year_total1`, sum(store_sales.`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
| FROM
| customer
- | INNER JOIN store_sales ON (`c_customer_sk` = `ss_customer_sk`)
- | INNER JOIN date_dim ON (`ss_sold_date_sk` = `d_date_sk`)
- | GROUP BY `c_customer_id`, `c_first_name`, `c_last_name`, `c_preferred_cust_flag`, `c_birth_country`, `c_login`, `c_email_address`, `d_year`, `d_date`) gen_subsumer_3
+ | INNER JOIN store_sales ON (customer.`c_customer_sk` = store_sales.`ss_customer_sk`)
+ | INNER JOIN date_dim ON (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`, date_dim.`d_date`) gen_subsumer_3
| WHERE
| false
| GROUP BY gen_subsumer_3.`customer_id`, gen_subsumer_3.`customer_first_name`, gen_subsumer_3.`customer_last_name`, gen_subsumer_3.`dyear`, gen_subsumer_3.`customer_preferred_cust_flag`, gen_subsumer_3.`customer_birth_country`, gen_subsumer_3.`customer_login`, gen_subsumer_3.`customer_email_address`
@@ -1262,10 +1451,7 @@ object TestTPCDS_1_4_Batch {
| limit 100
""".stripMargin.trim,
"""
- |
- |
- |
- """.stripMargin.trim),
+ """.stripMargin.trim),
//q74
("case_20",
"""
@@ -1346,9 +1532,9 @@ object TestTPCDS_1_4_Batch {
"""
|SELECT gen_subquery_1.`customer_id`, gen_subquery_1.`customer_first_name`, gen_subquery_1.`customer_last_name`
|FROM
- | (SELECT gen_subsumer_0.`customer_id` AS `customer_id`, gen_subsumer_0.`year_total_74` AS `year_total`
+ | (SELECT gen_subsumer_0.`customer_id` AS `customer_id`, sum(gen_subsumer_0.`year_total_74`) AS `year_total`
| FROM
- | (SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, date_dim.`d_year` AS `dyear`, date_dim.`d_date` AS `ddate`, date_dim.`d_month_seq`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, makedecimal(sum(unscaledvalue(store_sales.`ss_net_paid`))) AS `year_total_74`, 's' AS `sale_type`
+ | (SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, date_dim.`d_year` AS `dyear`, date_dim.`d_date` AS `ddate`, date_dim.`d_month_seq`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum(store_sales.`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
| FROM
| customer
| INNER JOIN store_sales ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`)
@@ -1357,9 +1543,9 @@ object TestTPCDS_1_4_Batch {
| WHERE
| (gen_subsumer_0.`dyear` IN (2001, 2002)) AND (gen_subsumer_0.`dyear` = 2001)
| GROUP BY gen_subsumer_0.`customer_id`, gen_subsumer_0.`customer_first_name`, gen_subsumer_0.`customer_last_name`, gen_subsumer_0.`dyear`
- | HAVING (gen_subsumer_0.`year_total_74` > 0.00BD)
+ | HAVING (sum(gen_subsumer_0.`year_total_74`) > 0.00BD)
| UNION ALL
- | SELECT customer.`c_customer_id` AS `customer_id`, makedecimal(sum(unscaledvalue(web_sales.`ws_net_paid`))) AS `year_total`
+ | SELECT customer.`c_customer_id` AS `customer_id`, sum(web_sales.`ws_net_paid`) AS `year_total`
| FROM
| customer
| INNER JOIN web_sales ON (customer.`c_customer_sk` = web_sales.`ws_bill_customer_sk`)
@@ -1368,19 +1554,19 @@ object TestTPCDS_1_4_Batch {
| false
| GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, date_dim.`d_year`
| HAVING (`year_total` > 0.00BD)) gen_subquery_0
- | INNER JOIN (SELECT gen_subsumer_1.`customer_id` AS `customer_id`, gen_subsumer_1.`customer_first_name` AS `customer_first_name`, gen_subsumer_1.`customer_last_name` AS `customer_last_name`, gen_subsumer_1.`year_total_74` AS `year_total`
+ | INNER JOIN (SELECT gen_subsumer_1.`customer_id` AS `customer_id`, gen_subsumer_1.`customer_first_name` AS `customer_first_name`, gen_subsumer_1.`customer_last_name` AS `customer_last_name`, sum(gen_subsumer_1.`year_total_74`) AS `year_total`
| FROM
- | (SELECT customer.`customer_id` AS `customer_id`, customer.`customer_first_name` AS `customer_first_name`, customer.`customer_last_name` AS `customer_last_name`, customer.`customer_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`customer_birth_country` AS `customer_birth_country`, customer.`customer_login` AS `customer_login`, customer.`customer_email_address` AS `customer_email_address`, date_dim.`dyear` AS `dyear`, date_dim.`ddate` AS `ddate`, date_dim.`d_month_seq`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, makedecimal(sum(unscaledvalue(store_sales.`ss_net_paid`))) AS `year_total_74`, 's' AS `sale_type`
+ | (SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, customer.`c_preferred_cust_flag` AS `customer_preferred_cust_flag`, customer.`c_birth_country` AS `customer_birth_country`, customer.`c_login` AS `customer_login`, customer.`c_email_address` AS `customer_email_address`, date_dim.`d_year` AS `dyear`, date_dim.`d_date` AS `ddate`, date_dim.`d_month_seq`, sum((CAST((((CAST(store_sales.`ss_ext_list_price` AS DECIMAL(8,2)) - CAST(store_sales.`ss_ext_wholesale_cost` AS DECIMAL(8,2))) - CAST(store_sales.`ss_ext_discount_amt` AS DECIMAL(8,2))) + CAST(store_sales.`ss_ext_sales_price` AS DECIMAL(8,2))) AS DECIMAL(12,2)) / 2.00BD)) AS `year_total`, sum(store_sales.`ss_net_paid`) AS `year_total_74`, 's' AS `sale_type`
| FROM
| customer
| INNER JOIN store_sales ON (store_sales.`ss_customer_sk` = customer.`c_customer_sk`)
| INNER JOIN date_dim ON (store_sales.`ss_sold_date_sk` = date_dim.`d_date_sk`)
- | GROUP BY customer.`customer_id`, customer.`customer_first_name`, customer.`customer_last_name`, customer.`customer_preferred_cust_flag`, customer.`customer_birth_country`, customer.`customer_login`, customer.`customer_email_address`, date_dim.`dyear`, date_dim.`ddate`, date_dim.`d_month_seq`) gen_subsumer_1
+ | GROUP BY customer.`c_customer_id`, customer.`c_first_name`, customer.`c_last_name`, customer.`c_preferred_cust_flag`, customer.`c_birth_country`, customer.`c_login`, customer.`c_email_address`, date_dim.`d_year`, date_dim.`d_date`, date_dim.`d_month_seq`) gen_subsumer_1
| WHERE
| (gen_subsumer_1.`dyear` IN (2001, 2002)) AND (gen_subsumer_1.`dyear` = 2002)
| GROUP BY gen_subsumer_1.`customer_id`, gen_subsumer_1.`customer_first_name`, gen_subsumer_1.`customer_last_name`, gen_subsumer_1.`dyear`
| UNION ALL
- | SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, makedecimal(sum(unscaledvalue(web_sales.`ws_net_paid`))) AS `year_total`
+ | SELECT customer.`c_customer_id` AS `customer_id`, customer.`c_first_name` AS `customer_first_name`, customer.`c_last_name` AS `customer_last_name`, sum(web_sales.`ws_net_paid`) AS `year_total`
| FR
<TRUNCATED>
[14/50] [abbrv] carbondata git commit: [CARBONDATA-2569] Change the
strategy of Search mode throw exception and run sparkSQL
Posted by ja...@apache.org.
[CARBONDATA-2569] Change the strategy of Search mode throw exception and run sparkSQL
Search mode throw exception but test case pass, please check the jira.
This closes #2357
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/83ee2c45
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/83ee2c45
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/83ee2c45
Branch: refs/heads/carbonstore
Commit: 83ee2c45fc10b220605916abe133b7a250007fdc
Parents: 041603d
Author: xubo245 <xu...@huawei.com>
Authored: Fri Jun 1 11:50:52 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Jun 11 18:06:27 2018 +0530
----------------------------------------------------------------------
.../main/scala/org/apache/spark/sql/CarbonSession.scala | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/83ee2c45/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index 497f95a..93c0b4a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -37,9 +37,8 @@ import org.apache.spark.sql.profiler.{Profiler, SQLStart}
import org.apache.spark.util.{CarbonReflectionUtils, Utils}
import org.apache.carbondata.common.annotations.InterfaceAudience
-import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.scan.expression.LiteralExpression
import org.apache.carbondata.core.util.{CarbonProperties, CarbonSessionInfo, ThreadLocalSessionInfo}
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
import org.apache.carbondata.store.SparkCarbonStore
@@ -101,8 +100,8 @@ class CarbonSession(@transient val sc: SparkContext,
} catch {
case e: Exception =>
logError(String.format(
- "Exception when executing search mode: %s, fallback to SparkSQL", e.getMessage))
- new Dataset[Row](self, qe, RowEncoder(qe.analyzed.schema))
+ "Exception when executing search mode: %s", e.getMessage))
+ throw e;
}
} else {
new Dataset[Row](self, qe, RowEncoder(qe.analyzed.schema))
@@ -171,19 +170,24 @@ class CarbonSession(@transient val sc: SparkContext,
*/
private def trySearchMode(qe: QueryExecution, sse: SQLStart): DataFrame = {
val analyzed = qe.analyzed
+ val LOG: LogService = LogServiceFactory.getLogService(this.getClass.getName)
analyzed match {
case _@Project(columns, _@Filter(expr, s: SubqueryAlias))
if s.child.isInstanceOf[LogicalRelation] &&
s.child.asInstanceOf[LogicalRelation].relation
.isInstanceOf[CarbonDatasourceHadoopRelation] =>
+ LOG.info(s"Search service started and supports filter: ${sse.sqlText}")
runSearch(analyzed, columns, expr, s.child.asInstanceOf[LogicalRelation])
case gl@GlobalLimit(_, ll@LocalLimit(_, p@Project(columns, _@Filter(expr, s: SubqueryAlias))))
if s.child.isInstanceOf[LogicalRelation] &&
s.child.asInstanceOf[LogicalRelation].relation
.isInstanceOf[CarbonDatasourceHadoopRelation] =>
val logicalRelation = s.child.asInstanceOf[LogicalRelation]
+ LOG.info(s"Search service started and supports limit: ${sse.sqlText}")
runSearch(analyzed, columns, expr, logicalRelation, gl.maxRows, ll.maxRows)
case _ =>
+ LOG.info(s"Search service started, but don't support: ${sse.sqlText}," +
+ s" and will run it with SparkSQL")
new Dataset[Row](self, qe, RowEncoder(qe.analyzed.schema))
}
}
[36/50] [abbrv] carbondata git commit: Support adding local
dictionary configuration in create table statement and show the configs in
describe formatted table
Posted by ja...@apache.org.
Support adding local dictionary configuration in create table statement and show the configs in describe formatted table
What changes were proposed in this pull request?
In this PR, in order to support local dictionary,
create table changes are made to support local dictionary configurations as table properties
show local dictionary properties in describe formatted command based on whether the local dictionary enabled or disabled.
Highlights:
basically we will have four properties
LOCAL_DICT_ENABLE => whether to enable or disable local dictionary
LOCAL_DICT_THRESHOLD => threshold property for the column to generate local dictionary
LOCAL_DICT_INCLUDE => columns for which local dictionary needs to be generated
LOCAL_DICT_EXCLUDE => columns for which local dictionary should not be generated
This closes#2375
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/be20fefb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/be20fefb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/be20fefb
Branch: refs/heads/carbonstore
Commit: be20fefbe0a01a501a567683ecc872e08f3fc001
Parents: ca466d9
Author: akashrn5 <ak...@gmail.com>
Authored: Wed Jun 6 20:33:39 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Tue Jun 19 21:08:27 2018 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 31 +++++
.../ThriftWrapperSchemaConverterImpl.java | 5 +
.../core/metadata/schema/table/CarbonTable.java | 68 +++++++++
.../schema/table/column/ColumnSchema.java | 19 +++
.../apache/carbondata/core/util/CarbonUtil.java | 108 ++++++++++++++
.../describeTable/TestDescribeTable.scala | 4 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 2 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 139 ++++++++++++++++++-
.../command/carbonTableSchemaCommon.scala | 13 +-
.../table/CarbonDescribeFormattedCommand.scala | 38 ++++-
10 files changed, 417 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/be20fefb/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 355bcb6..5f06d08 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -910,6 +910,37 @@ public final class CarbonCommonConstants {
public static final String COLUMN_GROUPS = "column_groups";
public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
public static final String DICTIONARY_INCLUDE = "dictionary_include";
+
+ /**
+ * Table property to enable or disable local dictionary generation
+ */
+ public static final String LOCAL_DICTIONARY_ENABLE = "local_dictionary_enable";
+
+ /**
+ * default value for local dictionary generation
+ */
+ public static final String LOCAL_DICTIONARY_ENABLE_DEFAULT = "true";
+
+ /**
+ * Threshold value for local dictionary
+ */
+ public static final String LOCAL_DICTIONARY_THRESHOLD = "local_dictionary_threshold";
+
+ /**
+ * default value for local dictionary
+ */
+ public static final String LOCAL_DICTIONARY_THRESHOLD_DEFAULT = "1000";
+
+ /**
+ * Table property to specify the columns for which local dictionary needs to be generated.
+ */
+ public static final String LOCAL_DICTIONARY_INCLUDE = "local_dictionary_include";
+
+ /**
+ * Table property to specify the columns for which local dictionary should not be to be generated.
+ */
+ public static final String LOCAL_DICTIONARY_EXCLUDE = "local_dictionary_exclude";
+
/**
* key for dictionary path
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/be20fefb/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index f03b997..12f5fc3 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -37,6 +37,7 @@ import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.metadata.schema.table.TableSchema;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation;
+import org.apache.carbondata.core.util.CarbonUtil;
/**
* Thrift schema to carbon schema converter and vice versa
@@ -594,6 +595,10 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
.getTable_columns()) {
listOfColumns.add(fromExternalToWrapperColumnSchema(externalColumnSchema));
}
+ if (null != externalTableSchema.tableProperties) {
+ CarbonUtil
+ .setLocalDictColumnsToWrapperSchema(listOfColumns, externalTableSchema.tableProperties);
+ }
wrapperTableSchema.setListOfColumns(listOfColumns);
wrapperTableSchema.setSchemaEvolution(
fromExternalToWrapperSchemaEvolution(externalTableSchema.getSchema_evolution()));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/be20fefb/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index f48ada0..b7bef28 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -151,6 +151,16 @@ public class CarbonTable implements Serializable {
private boolean hasDataMapSchema;
/**
+ * is local dictionary generation enabled for the table
+ */
+ private boolean isLocalDictionaryEnabled;
+
+ /**
+ * local dictionary generation threshold
+ */
+ private int localDictionaryThreshold;
+
+ /**
* The boolean field which points if the data written for Non Transactional Table
* or Transactional Table.
* transactional table means carbon will provide transactional support when user doing data
@@ -468,6 +478,37 @@ public class CarbonTable implements Serializable {
}
/**
+ * is local dictionary enabled for the table
+ * @return
+ */
+ public boolean isLocalDictionaryEnabled() {
+ return isLocalDictionaryEnabled;
+ }
+
+ /**
+ * set whether local dictionary enabled or not
+ * @param localDictionaryEnabled
+ */
+ public void setLocalDictionaryEnabled(boolean localDictionaryEnabled) {
+ isLocalDictionaryEnabled = localDictionaryEnabled;
+ }
+
+ /**
+ * @return local dictionary generation threshold
+ */
+ public int getLocalDictionaryThreshold() {
+ return localDictionaryThreshold;
+ }
+
+ /**
+ * set the local dictionary generation threshold
+ * @param localDictionaryThreshold
+ */
+ public void setLocalDictionaryThreshold(int localDictionaryThreshold) {
+ this.localDictionaryThreshold = localDictionaryThreshold;
+ }
+
+ /**
* build table unique name
* all should call this method to build table unique name
* @param databaseName
@@ -1045,5 +1086,32 @@ public class CarbonTable implements Serializable {
}
table.hasDataMapSchema =
null != tableInfo.getDataMapSchemaList() && tableInfo.getDataMapSchemaList().size() > 0;
+ setLocalDictInfo(table, tableInfo);
+ }
+
+ /**
+ * This method sets whether the local dictionary is enabled or not, and the local dictionary
+ * threshold, if not defined default value are considered.
+ * @param table
+ * @param tableInfo
+ */
+ private static void setLocalDictInfo(CarbonTable table, TableInfo tableInfo) {
+ String isLocalDictionaryEnabled = tableInfo.getFactTable().getTableProperties()
+ .get(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE);
+ String localDictionaryThreshold = tableInfo.getFactTable().getTableProperties()
+ .get(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD);
+ if (null != isLocalDictionaryEnabled) {
+ table.setLocalDictionaryEnabled(Boolean.parseBoolean(isLocalDictionaryEnabled));
+ if (null != localDictionaryThreshold) {
+ table.setLocalDictionaryThreshold(Integer.parseInt(localDictionaryThreshold));
+ } else {
+ table.setLocalDictionaryThreshold(
+ Integer.parseInt(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT));
+ }
+ } else {
+ // in case of old tables, local dictionary enable property will not be present in
+ // tableProperties, so disable the local dictionary generation
+ table.setLocalDictionaryEnabled(Boolean.parseBoolean("false"));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/be20fefb/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index fb4d8e3..786e873 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -138,6 +138,25 @@ public class ColumnSchema implements Serializable, Writable {
private String timeSeriesFunction = "";
/**
+ * set whether the column is local dictionary column or not.
+ */
+ private boolean isLocalDictColumn = false;
+
+ /**
+ * @return isLocalDictColumn
+ */
+ public boolean isLocalDictColumn() {
+ return isLocalDictColumn;
+ }
+
+ /**
+ * @param localDictColumn whether column is local dictionary column
+ */
+ public void setLocalDictColumn(boolean localDictColumn) {
+ isLocalDictColumn = localDictColumn;
+ }
+
+ /**
* @return the columnName
*/
public String getColumnName() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/be20fefb/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 836b193..2f34163 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -3004,5 +3004,113 @@ public final class CarbonUtil {
}
return blockId;
}
+
+ /**
+ * sets the local dictionary columns to wrapper schema, if the table property
+ * local_dictionary_include is defined, then those columns will be set as local dictionary
+ * columns, if not, all the no dictionary string datatype columns are set as local dictionary
+ * columns.
+ * Handling for complexTypes::
+ * Since the column structure will be flat
+ * if the parent column is configured as local Dictionary column, then it gets the child column
+ * count and then sets the primitive child column as local dictionary column if it is string
+ * datatype column
+ * Handling for both localDictionary Include and exclude columns:
+ * There will be basically four scenarios which are
+ * -------------------------------------------------------
+ * | Local_Dictionary_include | Local_Dictionary_Exclude |
+ * -------------------------------------------------------
+ * | Not Defined | Not Defined |
+ * | Not Defined | Defined |
+ * | Defined | Not Defined |
+ * | Defined | Defined |
+ * -------------------------------------------------------
+ * 1. when the both local dictionary include and exclude is not defined, then set all the no
+ * dictionary string datatype columns as local dictionary generate columns
+ * 2. set all the no dictionary string datatype columns as local dictionary columns except the
+ * columns present in local dictionary exclude
+ * 3. & 4. when local dictionary include is defined, no need to check dictionary exclude columns
+ * configured or not, we just need to set only the columns present in local dictionary include as
+ * local dictionary columns
+ *
+ * @param columns
+ * @param mainTableProperties
+ */
+ public static void setLocalDictColumnsToWrapperSchema(List<ColumnSchema> columns,
+ Map<String, String> mainTableProperties) {
+ String isLocalDictEnabledForMainTable =
+ mainTableProperties.get(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE);
+ String localDictIncludeColumnsOfMainTable =
+ mainTableProperties.get(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE);
+ String localDictExcludeColumnsOfMainTable =
+ mainTableProperties.get(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE);
+ String[] listOfDictionaryIncludeColumns = null;
+ String[] listOfDictionaryExcludeColumns = null;
+ if (null != isLocalDictEnabledForMainTable && Boolean
+ .parseBoolean(isLocalDictEnabledForMainTable)) {
+ int childColumnCount = 0;
+ for (ColumnSchema column : columns) {
+ // for complex type columns, user gives the parent column as local dictionary column and
+ // only the string primitive type child column will be set as local dictionary column in the
+ // schema
+ if (childColumnCount > 0) {
+ if (column.getDataType().equals(DataTypes.STRING)) {
+ column.setLocalDictColumn(true);
+ childColumnCount -= 1;
+ } else {
+ childColumnCount -= 1;
+ }
+ }
+ // if complex column is defined in local dictionary include column, then get the child
+ // columns and set the string datatype child type as local dictionary column
+ if (column.getNumberOfChild() > 0 && null != localDictIncludeColumnsOfMainTable) {
+ listOfDictionaryIncludeColumns = localDictIncludeColumnsOfMainTable.split(",");
+ for (String dictColumn : listOfDictionaryIncludeColumns) {
+ if (dictColumn.trim().equalsIgnoreCase(column.getColumnName())) {
+ childColumnCount = column.getNumberOfChild();
+ }
+ }
+ }
+ if (null == localDictIncludeColumnsOfMainTable) {
+ // if local dictionary exclude columns is not defined, then set all the no dictionary
+ // string datatype column
+ if (null == localDictExcludeColumnsOfMainTable) {
+ // column should be no dictionary string datatype column
+ if (column.isDimensionColumn() && column.getDataType().equals(DataTypes.STRING)
+ && !column.hasEncoding(Encoding.DICTIONARY)) {
+ column.setLocalDictColumn(true);
+ }
+ // if local dictionary exclude columns is defined, then set for all no dictionary string
+ // datatype columns except excluded columns
+ } else {
+ if (column.isDimensionColumn() && column.getDataType().equals(DataTypes.STRING)
+ && !column.hasEncoding(Encoding.DICTIONARY)) {
+ listOfDictionaryExcludeColumns = localDictExcludeColumnsOfMainTable.split(",");
+ for (String excludeDictColumn : listOfDictionaryExcludeColumns) {
+ if (!excludeDictColumn.trim().equalsIgnoreCase(column.getColumnName())) {
+ column.setLocalDictColumn(true);
+ }
+ }
+ }
+ }
+ } else {
+ // if local dict columns alre not configured, set for all no dictionary string datatype
+ // column
+ if (column.isDimensionColumn() && column.getDataType().equals(DataTypes.STRING) && !column
+ .hasEncoding(Encoding.DICTIONARY) && localDictIncludeColumnsOfMainTable.toLowerCase()
+ .contains(column.getColumnName().toLowerCase())) {
+ if (null == listOfDictionaryIncludeColumns) {
+ listOfDictionaryIncludeColumns = localDictIncludeColumnsOfMainTable.split(",");
+ }
+ for (String dictColumn : listOfDictionaryIncludeColumns) {
+ if (dictColumn.trim().equalsIgnoreCase(column.getColumnName())) {
+ column.setLocalDictColumn(true);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/be20fefb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
index 1e333ee..5598457 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
@@ -51,10 +51,10 @@ class TestDescribeTable extends QueryTest with BeforeAndAfterAll {
test("test describe formatted table desc1") {
val resultCol = Seq("", "", "##Detailed Column property", "##Detailed Table Information", "ADAPTIVE", "CARBON Store Path", "Comment", "Database Name", "Last Update Time",
- "SORT_COLUMNS", "SORT_SCOPE", "Streaming", "Table Block Size", "Table Data Size", "Table Index Size", "Table Name", "dec2col1", "dec2col2", "dec2col3", "dec2col4")
+ "SORT_COLUMNS", "SORT_SCOPE", "Streaming", "Table Block Size", "Local Dictionary Enabled", "Local Dictionary Threshold","Table Data Size", "Table Index Size", "Table Name", "dec2col1", "dec2col2", "dec2col3", "dec2col4")
val resultRow: Seq[Row] = resultCol map(propName => Row(f"$propName%-36s"))
checkAnswer(sql("desc formatted DESC1").select("col_name"), resultRow)
- assert(sql("desc formatted desc1").count() == 20)
+ assert(sql("desc formatted desc1").count() == 22)
}
test("test describe formatted for partition table") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/be20fefb/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index b43ae3f..1ccbf6a 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -44,7 +44,7 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.metadata.ColumnIdentifier
import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, StructField => CarbonStructField}
import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema, DataMapSchemaStorageProvider}
+import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema}
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, ColumnSchema}
import org.apache.carbondata.core.util.DataTypeUtil
import org.apache.carbondata.processing.exception.DataLoadingException
http://git-wip-us.apache.org/repos/asf/carbondata/blob/be20fefb/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 7d28790..65ff76d 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -24,6 +24,7 @@ import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.{ArrayBuffer, LinkedHashSet, Map}
import scala.language.implicitConversions
+import scala.util.Try
import scala.util.matching.Regex
import org.apache.hadoop.hive.ql.lib.Node
@@ -43,7 +44,7 @@ import org.apache.carbondata.core.metadata.schema.partition.PartitionType
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.processing.util.CarbonLoaderUtil
-import org.apache.carbondata.spark.util.{CommonUtil, DataTypeConverterUtil}
+import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, DataTypeConverterUtil}
/**
* TODO remove the duplicate code and add the common methods to common class.
@@ -292,6 +293,83 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
s"${CarbonCommonConstants.COLUMN_GROUPS} is deprecated")
}
+ // validate the local dictionary property if defined
+ if (tableProperties.get(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE).isDefined) {
+ Try(tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE).toBoolean) match {
+ case scala.util.Success(value) =>
+ case scala.util.Failure(ex) =>
+ tableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE,
+ CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE_DEFAULT)
+ }
+ } else {
+ // if LOCAL_DICTIONARY_ENABLE is not defined, consider the default value which is true
+ tableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE,
+ CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE_DEFAULT)
+ }
+
+ // validate the local dictionary threshold property if defined
+ if (tableProperties.get(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD).isDefined) {
+ // if any invalid value is configured for LOCAL_DICTIONARY_THRESHOLD, then default value
+ // will be
+ // considered which is 1000
+ Try(tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD).toInt) match {
+ case scala.util.Success(value) =>
+ if (value <= 0) {
+ tableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD,
+ CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT)
+ }
+ case scala.util.Failure(ex) =>
+ LOGGER
+ .debug(
+ "invalid value is configured for local_dictionary_threshold, considering the defaut" +
+ " " +
+ "value")
+ tableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD,
+ CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT)
+ }
+ }
+
+ // validate the local dictionary columns defined, this we will validated if the local dictionary
+ // is enabled, else it is not validated
+ if (!(tableProperties.get(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE).isDefined &&
+ tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE).trim
+ .equalsIgnoreCase("false"))) {
+ var localDictIncludeColumns: Seq[String] = Seq[String]()
+ var localDictExcludeColumns: Seq[String] = Seq[String]()
+ val isLocalDictIncludeDefined = tableProperties
+ .get(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE)
+ .isDefined
+ val isLocalDictExcludeDefined = tableProperties
+ .get(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE)
+ .isDefined
+ if (isLocalDictIncludeDefined) {
+ localDictIncludeColumns =
+ tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE).split(",").map(_.trim)
+ // validate all the local dictionary include columns
+ validateLocalDictionaryColumns(fields, tableProperties, localDictIncludeColumns)
+ }
+ if (isLocalDictExcludeDefined) {
+ localDictExcludeColumns =
+ tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE).split(",").map(_.trim)
+ // validate all the local dictionary exclude columns
+ validateLocalDictionaryColumns(fields, tableProperties, localDictExcludeColumns)
+ }
+ // validate if both local dictionary include and exclude contains same column
+ if (isLocalDictIncludeDefined && isLocalDictExcludeDefined) {
+ val localDictIncludeCols = tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE)
+ val localDictExcludeCols = tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE)
+ if (List(localDictIncludeCols, localDictExcludeCols).mkString(",").split(",")
+ .distinct.length !=
+ List(localDictIncludeCols, localDictExcludeCols).mkString(",").split(",")
+ .length) {
+ val errMsg =
+ "Column ambiguity as duplicate columns present in LOCAL_DICTIONARY_INCLUDE and " +
+ "LOCAL_DICTIONARY_INCLUDE.Duplicate columns are not allowed."
+ throw new MalformedCarbonCommandException(errMsg)
+ }
+ }
+ }
+
// get no inverted index columns from table properties.
val noInvertedIdxCols = extractNoInvertedIndexColumns(fields, tableProperties)
// get partitionInfo
@@ -322,6 +400,65 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
/**
+ * This method validates the local dictionary configured columns
+ *
+ * @param fields
+ * @param tableProperties
+ */
+ private def validateLocalDictionaryColumns(fields: Seq[Field],
+ tableProperties: Map[String, String], localDictColumns: Seq[String]): Unit = {
+ var dictIncludeColumns: Seq[String] = Seq[String]()
+
+ // check if the duplicate columns are specified in table schema
+ if (localDictColumns.distinct.lengthCompare(localDictColumns.size) != 0) {
+ val a = localDictColumns.diff(localDictColumns.distinct).distinct
+ val errMsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE contains Duplicate Columns " +
+ a.mkString("(", ",", ")") +
+ ". Please check create table statement."
+ throw new MalformedCarbonCommandException(errMsg)
+ }
+
+ // check if the column specified exists in table schema
+ localDictColumns.foreach { distCol =>
+ if (!fields.exists(x => x.column.equalsIgnoreCase(distCol.trim))) {
+ val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " + distCol.trim +
+ " does not exist in table. Please check create table statement."
+ throw new MalformedCarbonCommandException(errormsg)
+ }
+ }
+
+ // check if column is other than string datatype
+ localDictColumns.foreach { dictColm =>
+ if (fields
+ .exists(x => x.column.equalsIgnoreCase(dictColm) &&
+ !x.dataType.get.equalsIgnoreCase("STRING") &&
+ !x.dataType.get.equalsIgnoreCase("STRUCT") &&
+ !x.dataType.get.equalsIgnoreCase("ARRAY"))) {
+ val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " +
+ dictColm.trim +
+ " is not a String datatype column. LOCAL_DICTIONARY_COLUMN should be no " +
+ "dictionary string datatype column.Please check create table statement."
+ throw new MalformedCarbonCommandException(errormsg)
+ }
+ }
+ // check if the same column is present in both dictionary include and local dictionary columns
+ // configuration
+ if (tableProperties.get(CarbonCommonConstants.DICTIONARY_INCLUDE).isDefined) {
+ dictIncludeColumns =
+ tableProperties(CarbonCommonConstants.DICTIONARY_INCLUDE).split(",").map(_.trim)
+ localDictColumns.foreach { distCol =>
+ if (dictIncludeColumns.exists(x => x.equalsIgnoreCase(distCol.trim))) {
+ val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " +
+ distCol.trim +
+ " specified in Dictionary include. Local Dictionary will not be " +
+ "generated for Dictionary include. Please check create table statement."
+ throw new MalformedCarbonCommandException(errormsg)
+ }
+ }
+ }
+ }
+
+ /**
* Extract the column groups configuration from table properties.
* Based on this Row groups of fields will be determined.
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/be20fefb/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index aa40a1f..d48db21 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -27,7 +27,6 @@ import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.util.CarbonException
-import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.Segment
@@ -40,7 +39,7 @@ import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, RelationId
import org.apache.carbondata.core.metadata.schema.table.column.{ColumnSchema, ParentColumnTableRelation}
import org.apache.carbondata.core.service.impl.ColumnUniqueIdGenerator
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentUpdateStatusManager}
-import org.apache.carbondata.core.util.DataTypeUtil
+import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
import org.apache.carbondata.processing.loading.FailureCauses
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.merger.CompactionType
@@ -246,7 +245,6 @@ class AlterTableColumnSchemaGenerator(
allColumns ++= Seq(columnSchema)
newCols ++= Seq(columnSchema)
})
-
allColumns ++= tableCols.filter(x => !x.isDimensionColumn)
alterTableModel.msrCols.foreach(field => {
val encoders = new java.util.ArrayList[Encoding]()
@@ -294,7 +292,9 @@ class AlterTableColumnSchemaGenerator(
alterTableModel.tableProperties.foreach {
x => val value = tablePropertiesMap.get(x._1)
if (null != value) {
- tablePropertiesMap.put(x._1, value + "," + x._2)
+ if (value != x._2) {
+ tablePropertiesMap.put(x._1, value + "," + x._2)
+ }
} else {
tablePropertiesMap.put(x._1, x._2)
}
@@ -550,6 +550,11 @@ class TableNewProcessor(cm: TableModel) {
}
}
+ // check whether the column is a local dictionary column and set in column schema
+ if (null != cm.tableProperties) {
+ CarbonUtil
+ .setLocalDictColumnsToWrapperSchema(allColumns.asJava, cm.tableProperties.asJava)
+ }
cm.msrCols.foreach { field =>
// if aggregate function is defined in case of preaggregate and agg function is sum or avg
// then it can be stored as measure
http://git-wip-us.apache.org/repos/asf/carbondata/blob/be20fefb/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
index 617f5e8..c6bd567 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
@@ -30,8 +30,7 @@ import org.codehaus.jackson.map.ObjectMapper
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
+import org.apache.carbondata.core.util.CarbonUtil
private[sql] case class CarbonDescribeFormattedCommand(
child: SparkPlan,
@@ -111,6 +110,41 @@ private[sql] case class CarbonDescribeFormattedCommand(
.LOAD_SORT_SCOPE_DEFAULT)))
val isStreaming = tblProps.asScala.getOrElse("streaming", "false")
results ++= Seq(("Streaming", isStreaming, ""))
+ val isLocalDictEnabled = tblProps.asScala
+ .getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE,
+ CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE_DEFAULT)
+ results ++= Seq(("Local Dictionary Enabled", isLocalDictEnabled, ""))
+ // if local dictionary is enabled, then only show other properties of local dictionary
+ if (isLocalDictEnabled.toBoolean) {
+ val localDictThreshold = tblProps.asScala
+ .getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD,
+ CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT)
+ results ++= Seq(("Local Dictionary Threshold", localDictThreshold, ""))
+ if (tblProps.asScala
+ .get(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE).isDefined) {
+ val allLocalDictColumns = tblProps.asScala(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE)
+ .split(",")
+ results ++= Seq(("Local Dictionary Include", getDictColumnString(allLocalDictColumns), ""))
+ }
+ if (tblProps.asScala
+ .get(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE).isDefined) {
+ val allLocalDictColumns = tblProps.asScala(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE)
+ .split(",")
+ results ++= Seq(("Local Dictionary Exclude", getDictColumnString(allLocalDictColumns), ""))
+ }
+ }
+
+ /**
+ * return the string which has all comma separated columns
+ * @param localDictColumns
+ * @return
+ */
+ def getDictColumnString(localDictColumns: Array[String]): String = {
+ val dictColumns: StringBuilder = new StringBuilder
+ localDictColumns.foreach(column => dictColumns.append(column).append(","))
+ dictColumns.toString().patch(dictColumns.toString().lastIndexOf(","), "", 1)
+ }
+
// show table level compaction options
if (tblProps.containsKey(CarbonCommonConstants.TABLE_MAJOR_COMPACTION_SIZE)) {
[38/50] [abbrv] carbondata git commit: [CARBONDATA-2611] Added Test
Cases for Local Dictionary Support for Create Table comand
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c5a4ec07/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
new file mode 100644
index 0000000..5f9af69
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
@@ -0,0 +1,2102 @@
+package org.apache.carbondata.spark.testsuite.localdictionary
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+
+class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfterAll {
+
+ override protected def beforeAll(): Unit = {
+ sql("DROP TABLE IF EXISTS LOCAL1")
+ }
+
+ test("test local dictionary default configuration") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ val desc_result = sql("describe formatted local1")
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict columns _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='name')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations for local dict columns _002")
+ {
+ sql("drop table if exists local1")
+
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict columns _003") {
+ sql("drop table if exists local1")
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='')
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " +
+ "create table statement"))
+ }
+
+ test("test local dictionary custom configurations for local dict columns _004") {
+ sql("drop table if exists local1")
+ val exception1 = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='abc')
+ """.
+ stripMargin)
+ }
+ assert(exception1.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "create table " +
+ "statement"))
+ }
+
+ test("test local dictionary custom configurations for local dict columns _005") {
+ sql("drop table if exists local1")
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='id')
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_COLUMN should " +
+ "be no dictionary string/complex datatype column"))
+ }
+
+ test("test local dictionary custom configurations for local dict columns _006") {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('dictionary_include'='name','local_dictionary_include'='name')
+ """.
+ stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict threshold _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='10000')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict threshold _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='-100')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict threshold _003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='21474874811')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict threshold _004")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict threshold _005")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='hello')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_001")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='-100','local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='','local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_004")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='vdslv','local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_005")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_006")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'=' ')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_007")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'='hello')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_008")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'='name',
+ | 'dictionary_include'='name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_009")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='','local_dictionary_include'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_010")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='-100','local_dictionary_include'='Hello')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_011")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='23213497321591234324',
+ | 'local_dictionary_include'='name','dictionary_include'='name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary default configuration when enabled") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true')
+ """.stripMargin)
+
+ val desc_result = sql("describe formatted local1")
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict columns _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='name','local_dictionary_enable'='true')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict columns _002")
+ {
+ sql("drop table if exists local1")
+
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict columns _003") {
+ sql("drop table if exists local1")
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='')
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " +
+ "create table statement"))
+
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict columns _004") {
+ sql("drop table if exists local1")
+ val exception1 = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='abc')
+ """.
+ stripMargin)
+ }
+ assert(exception1.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "create table " +
+ "statement"))
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict columns _005") {
+ sql("drop table if exists local1")
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='id')
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_COLUMN should " +
+ "be no dictionary string/complex datatype column"))
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict columns _006") {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','dictionary_include'='name',
+ | 'local_dictionary_include'='name')
+ """.
+ stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations when local_dictionary_exclude is configured _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_enable'='true')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test local dictionary custom configurations when local_dictionary_exclude is configured _002")
+ {
+ sql("drop table if exists local1")
+
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations when local_dictionary_exclude is configured _003") {
+ sql("drop table if exists local1")
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='')
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " +
+ "create table statement"))
+
+ }
+
+ test("test local dictionary custom configurations when local_dictionary_exclude is configured _004") {
+ sql("drop table if exists local1")
+ val exception1 = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='abc')
+ """.
+ stripMargin)
+ }
+ assert(exception1.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "create table " +
+ "statement"))
+ }
+
+ test("test local dictionary custom configurations when local_dictionary_exclude is configured _005") {
+ sql("drop table if exists local1")
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='id')
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_COLUMN should " +
+ "be no dictionary string/complex datatype column"))
+ }
+
+ test("test local dictionary custom configurations when local_dictionary_exclude is configured _006") {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','dictionary_include'='name',
+ | 'local_dictionary_exclude'='name')
+ """.
+ stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _001")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city',
+ | 'local_dictionary_enable'='true')
+ """.
+ stripMargin)
+
+ val descFormatted1 = sql("describe formatted local1").collect
+
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _002") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int,add string)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city','sort_columns'='add',
+ | 'local_dictionary_enable'='true')
+ """.
+ stripMargin)
+
+ val descFormatted1 = sql("describe formatted local1").collect
+
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city',
+ | 'local_dictionary_enable'='false')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+
+ checkExistence(sql("describe formatted local1"), false, "Local Dictionary Include")
+ checkExistence(sql("describe formatted local1"), false, "Local Dictionary Exclude")
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _004")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city',
+ | 'local_dictionary_enable'='true','dictionary_include'='name,city')
+ """.
+ stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _005")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name,city',
+ | 'local_dictionary_exclude'='name')
+ """.
+ stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _006")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int,st struct<s_id:int,
+ | s_name:string,s_city:array<string>>)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city,st',
+ | 'local_dictionary_enable'='true')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city,st"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _007")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int,st array<struct<s_id:int,
+ | s_name:string>>)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city,st',
+ | 'local_dictionary_enable'='true')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city,st"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict threshold _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict threshold _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict threshold _003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='21474874811')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict threshold _004")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict threshold _005")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='hello')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _001")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _004")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='vdslv',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _005")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _006")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'=' ')
+ """.stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _007")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='hello')
+ """.stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _008")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='name','dictionary_include'='name')
+ """.stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _009")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='',
+ | 'local_dictionary_include'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _010")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100',
+ | 'local_dictionary_include'='Hello')
+ """.stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _011")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true',
+ | 'local_dictionary_threshold'='23213497321591234324','local_dictionary_include'='name',
+ | 'dictionary_include'='name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary default configuration when disabled") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+
+ val desc_result = sql("describe formatted local1")
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict columns _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='name','local_dictionary_enable'='false')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict columns _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_include'='name,name')
+ """.stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict columns _003") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_include'='')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict columns _004") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_include'='abc')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict columns _005") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_include'='id')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict columns _006") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','dictionary_include'='name',
+ | 'local_dictionary_include'='name')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict threshold _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict threshold _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='-100')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict threshold _003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='21474874811')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict threshold _004")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict threshold _005")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='hello')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _001")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='-100',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _004")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='vdslv',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _005")
+ {
+ sql("drop table if exists local1")
+
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='name,name')
+ """.stripMargin)
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _006")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'=' ')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _007")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='hello')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _008")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='name','dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _009")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='',
+ | 'local_dictionary_include'='name,name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _010")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='-100',
+ | 'local_dictionary_include'='Hello')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _011")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false',
+ | 'local_dictionary_threshold'='23213497321591234324','local_dictionary_include'='name',
+ | 'dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configuration with other table properties _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('dictionary_include'='city','sort_scope'='global_sort',
+ | 'sort_columns'='city,name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
+ case Some(row) => assert(row.get(1).toString.contains("global_sort"))
+ }
+ }
+
+ test("test local dictionary custom configuration with other table properties _002") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('dictionary_include'='city','sort_scope'='batch_sort',
+ | 'sort_columns'='city,name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
+ case Some(row) => assert(row.get(1).toString.contains("batch_sort"))
+ }
+ }
+ test("test local dictionary custom configuration with other table properties _003") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('dictionary_include'='city','sort_scope'='no_sort',
+ | 'sort_columns'='city,name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
+ case Some(row) => assert(row.get(1).toString.contains("no_sort"))
+ }
+ }
+ test("test local dictionary custom configuration with other table properties _004") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('dictionary_include'='city','sort_scope'='local_sort',
+ | 'sort_columns'='city,name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
+ case Some(row) => assert(row.get(1).toString.contains("local_sort"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary default configuration when enabled") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | create table local1 stored by 'carbondata' tblproperties('local_dictionary_enable'='true') as
+ | select * from local
+ """.stripMargin)
+
+ val desc_result = sql("describe formatted local1")
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _001") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='name','local_dictionary_enable'='true')
+ | as select * from local
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _002")
+ {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name,name')
+ | as select * from local
+ """.stripMargin)
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _003") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='')
+ | as select * from local
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " +
+ "create table statement"))
+
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _004") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ val exception1 = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='abc')
+ | as select * from local
+ """.
+ stripMargin)
+ }
+ assert(exception1.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "create table " +
+ "statement"))
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _005") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='id')
+ | as select * from local
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_COLUMN should " +
+ "be no dictionary string/complex datatype column"))
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _006") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','dictionary_include'='name',
+ | 'local_dictionary_include'='name') as select * from local
+ """.
+ stripMargin)
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _001") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_enable'='true')
+ | as select * from local
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _002")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='name,name')
+ | as select * from local
+ """.stripMargin)
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _003") {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='')
+ | as select * from local
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " +
+ "create table statement"))
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _004") {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ val exception1 = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='abc')
+ | as select * from local
+ """.
+ stripMargin)
+ }
+ assert(exception1.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "create table " +
+ "statement"))
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _005") {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='id')
+ | as select * from local
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_COLUMN should " +
+ "be no dictionary string/complex datatype column"))
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _006") {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','dictionary_include'='name',
+ | 'local_dictionary_exclude'='name') as select * from local
+ """.
+ stripMargin)
+ }
+ }
+
+ test(
+ "test CTAS statements for local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _001")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city',
+ | 'local_dictionary_enable'='true') as select * from local
+ """.
+ stripMargin)
+
+ val descFormatted1 = sql("describe formatted local1").collect
+
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test(
+ "test CTAS statements for local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _002")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city',
+ | 'local_dictionary_enable'='false') as select * from local
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+
+ checkExistence(sql("describe formatted local1"), false, "Local Dictionary Include")
+ checkExistence(sql("describe formatted local1"), false, "Local Dictionary Exclude")
+ }
+
+ test(
+ "test CTAS statements for local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _003")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city',
+ | 'local_dictionary_enable'='true','dictionary_include'='name,city') as select * from local
+ """.
+ stripMargin)
+ }
+ }
+
+ test(
+ "test CTAS statements for local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _004")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name,city',
+ | 'local_dictionary_exclude'='name') as select * from local
+ """.
+ stripMargin)
+ }
+ }
+
+ test(
+ "test CTAS statements for local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _005")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int,st struct<s_id:int,
+ | s_name:string,s_city:array<string>>)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city,st',
+ | 'local_dictionary_enable'='false') as select * from local
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict threshold _001") {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000')
+ | as select * from local
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict threshold _002")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100')
+ | as select * from local
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict threshold _003")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='23589714365172595')
+ | as select * from local
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when first table is hive table")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000','local_dictionary_include'='city')
+ | as select * from local
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test("test no inverted index for local dictionary custom configurations")
+ {
+ sql("drop table if exists local1")
+
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true',
+ | 'local_dictionary_threshold'='10000','local_dictionary_include'='city','no_inverted_index'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ override protected def afterAll(): Unit = {
+ sql("DROP TABLE IF EXISTS LOCAL1")
+ }
+}
[07/50] [abbrv] carbondata git commit: [Hoxfix] Upgrade dev version
to 1.5.0-SNAPSHOT and fix some small issues
Posted by ja...@apache.org.
[Hoxfix] Upgrade dev version to 1.5.0-SNAPSHOT and fix some small issues
1.Upgrade dev version to 1.5.0-SNAPSHOT
2.Fix carbon-spark-sql issue
3.Remove hadoop 2.2 profile
This closes #2359
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4bb7e278
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4bb7e278
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4bb7e278
Branch: refs/heads/carbonstore
Commit: 4bb7e2785f961b2697b7de2a3a2556c25a5bb6b3
Parents: 56bf4e4
Author: chenliang613 <ch...@huawei.com>
Authored: Sat Jun 2 12:58:57 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Jun 2 12:39:10 2018 +0530
----------------------------------------------------------------------
assembly/pom.xml | 2 +-
bin/carbon-spark-sql | 4 ++--
common/pom.xml | 2 +-
core/pom.xml | 2 +-
datamap/bloom/pom.xml | 2 +-
datamap/examples/pom.xml | 2 +-
datamap/lucene/pom.xml | 2 +-
datamap/mv/core/pom.xml | 2 +-
datamap/mv/plan/pom.xml | 2 +-
examples/flink/pom.xml | 2 +-
examples/spark2/pom.xml | 2 +-
format/pom.xml | 2 +-
hadoop/pom.xml | 2 +-
integration/hive/pom.xml | 2 +-
integration/presto/pom.xml | 2 +-
integration/spark-common-cluster-test/pom.xml | 2 +-
integration/spark-common-test/pom.xml | 2 +-
integration/spark-common/pom.xml | 2 +-
integration/spark2/pom.xml | 2 +-
pom.xml | 8 +-------
processing/pom.xml | 2 +-
store/sdk/pom.xml | 2 +-
store/search/pom.xml | 2 +-
streaming/pom.xml | 2 +-
24 files changed, 25 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 56522d0..eb3d3a9 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/bin/carbon-spark-sql
----------------------------------------------------------------------
diff --git a/bin/carbon-spark-sql b/bin/carbon-spark-sql
index 4b927d1..9916fef 100755
--- a/bin/carbon-spark-sql
+++ b/bin/carbon-spark-sql
@@ -36,13 +36,13 @@ export CARBON_SOURCE="$(cd "`dirname "$0"`"/..; pwd)"
ASSEMBLY_DIR="$CARBON_SOURCE/assembly/target/scala-2.11"
GREP_OPTIONS=
-num_jars="$(ls -1 "$ASSEMBLY_DIR" | grep "^carbondata.*hadoop.*\.jar$" | wc -l)"
+num_jars="$(ls -1 "$ASSEMBLY_DIR" | grep "^apache-carbondata.*\.jar$" | wc -l)"
if [ "$num_jars" -eq "0" -a -z "$ASSEMBLY_DIR" ]; then
echo "Failed to find Carbondata assembly in $ASSEMBLY_DIR." 1>&2
echo "You need to build Carbondata before running this program." 1>&2
exit 1
fi
-ASSEMBLY_JARS="$(ls -1 "$ASSEMBLY_DIR" | grep "^carbondata.*hadoop.*\.jar$" || true)"
+ASSEMBLY_JARS="$(ls -1 "$ASSEMBLY_DIR" | grep "^apache-carbondata.*\.jar$" || true)"
if [ "$num_jars" -gt "1" ]; then
echo "Found multiple Carbondata assembly jars in $ASSEMBLY_DIR:" 1>&2
echo "$ASSEMBLY_JARS" 1>&2
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 433d575..1209388 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index d9c756e..7d87037 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/datamap/bloom/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/bloom/pom.xml b/datamap/bloom/pom.xml
index b7db969..f13d477 100644
--- a/datamap/bloom/pom.xml
+++ b/datamap/bloom/pom.xml
@@ -6,7 +6,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/datamap/examples/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/examples/pom.xml b/datamap/examples/pom.xml
index 309828d..be65529 100644
--- a/datamap/examples/pom.xml
+++ b/datamap/examples/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/datamap/lucene/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/lucene/pom.xml b/datamap/lucene/pom.xml
index bdb8876..c5c7555 100644
--- a/datamap/lucene/pom.xml
+++ b/datamap/lucene/pom.xml
@@ -6,7 +6,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/datamap/mv/core/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/mv/core/pom.xml b/datamap/mv/core/pom.xml
index 99a8e22..54960a1 100644
--- a/datamap/mv/core/pom.xml
+++ b/datamap/mv/core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/datamap/mv/plan/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/pom.xml b/datamap/mv/plan/pom.xml
index 6a36fc5..fcf0e51 100644
--- a/datamap/mv/plan/pom.xml
+++ b/datamap/mv/plan/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/examples/flink/pom.xml
----------------------------------------------------------------------
diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml
index b783435..6af8f19 100644
--- a/examples/flink/pom.xml
+++ b/examples/flink/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/examples/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index 196bc16..7a55333 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/format/pom.xml
----------------------------------------------------------------------
diff --git a/format/pom.xml b/format/pom.xml
index 41197cf..039b0a0 100644
--- a/format/pom.xml
+++ b/format/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index 07883cd..1d7fab3 100644
--- a/hadoop/pom.xml
+++ b/hadoop/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/integration/hive/pom.xml
----------------------------------------------------------------------
diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml
index 68245db..c144353 100644
--- a/integration/hive/pom.xml
+++ b/integration/hive/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/integration/presto/pom.xml
----------------------------------------------------------------------
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index c2d941a..bfa05f9 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/integration/spark-common-cluster-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/pom.xml b/integration/spark-common-cluster-test/pom.xml
index d8aecc2..87e08d9 100644
--- a/integration/spark-common-cluster-test/pom.xml
+++ b/integration/spark-common-cluster-test/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/integration/spark-common-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/pom.xml b/integration/spark-common-test/pom.xml
index ae78523..b8629bf 100644
--- a/integration/spark-common-test/pom.xml
+++ b/integration/spark-common-test/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/integration/spark-common/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common/pom.xml b/integration/spark-common/pom.xml
index f011a75..599c6c8 100644
--- a/integration/spark-common/pom.xml
+++ b/integration/spark-common/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/integration/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml
index 73d48ef..2bce694 100644
--- a/integration/spark2/pom.xml
+++ b/integration/spark2/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e9551c0..1413fd1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -35,7 +35,7 @@
<inceptionYear>2016</inceptionYear>
<packaging>pom</packaging>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<licenses>
<license>
@@ -464,12 +464,6 @@
</build>
</profile>
<profile>
- <id>hadoop-2.2.0</id>
- <properties>
- <hadoop.version>2.2.0</hadoop.version>
- </properties>
- </profile>
- <profile>
<id>spark-2.1</id>
<properties>
<spark.version>2.1.0</spark.version>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/processing/pom.xml
----------------------------------------------------------------------
diff --git a/processing/pom.xml b/processing/pom.xml
index b1a103a..ab7c96c 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/store/sdk/pom.xml
----------------------------------------------------------------------
diff --git a/store/sdk/pom.xml b/store/sdk/pom.xml
index af0d079..fbeb562 100644
--- a/store/sdk/pom.xml
+++ b/store/sdk/pom.xml
@@ -7,7 +7,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/store/search/pom.xml
----------------------------------------------------------------------
diff --git a/store/search/pom.xml b/store/search/pom.xml
index 9d833f2..6acbbfb 100644
--- a/store/search/pom.xml
+++ b/store/search/pom.xml
@@ -7,7 +7,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/streaming/pom.xml
----------------------------------------------------------------------
diff --git a/streaming/pom.xml b/streaming/pom.xml
index b8c447d..0883f70 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -4,7 +4,7 @@
<parent>
<artifactId>carbondata-parent</artifactId>
<groupId>org.apache.carbondata</groupId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
[29/50] [abbrv] carbondata git commit: [CARBONDATA-2428] Support flat
folder for managed carbon table
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
index 99b536c..b4937e6 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
@@ -17,12 +17,15 @@
package org.apache.carbondata.spark.testsuite.datacompaction
+import scala.collection.JavaConverters._
+
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
-import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.writer.CarbonIndexFileMergeWriter
@@ -61,7 +64,7 @@ class CarbonIndexFileMergeTestCase
""".stripMargin)
sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE indexmerge OPTIONS('header'='false', " +
s"'GLOBAL_SORT_PARTITIONS'='100')")
- val table = CarbonMetadata.getInstance().getCarbonTable("default","indexmerge")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "indexmerge")
new CarbonIndexFileMergeWriter(table)
.mergeCarbonIndexFilesOfSegment("0", table.getTablePath, false)
assert(getIndexFileCount("default_indexmerge", "0") == 0)
@@ -84,7 +87,7 @@ class CarbonIndexFileMergeTestCase
val rows = sql("""Select count(*) from nonindexmerge""").collect()
assert(getIndexFileCount("default_nonindexmerge", "0") == 100)
assert(getIndexFileCount("default_nonindexmerge", "1") == 100)
- val table = CarbonMetadata.getInstance().getCarbonTable("default","nonindexmerge")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "nonindexmerge")
new CarbonIndexFileMergeWriter(table)
.mergeCarbonIndexFilesOfSegment("0", table.getTablePath, false)
new CarbonIndexFileMergeWriter(table)
@@ -109,7 +112,7 @@ class CarbonIndexFileMergeTestCase
val rows = sql("""Select count(*) from nonindexmerge""").collect()
assert(getIndexFileCount("default_nonindexmerge", "0") == 100)
assert(getIndexFileCount("default_nonindexmerge", "1") == 100)
- val table = CarbonMetadata.getInstance().getCarbonTable("default","nonindexmerge")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "nonindexmerge")
new CarbonIndexFileMergeWriter(table)
.mergeCarbonIndexFilesOfSegment("0", table.getTablePath, false)
new CarbonIndexFileMergeWriter(table)
@@ -138,7 +141,7 @@ class CarbonIndexFileMergeTestCase
assert(getIndexFileCount("default_nonindexmerge", "1") == 100)
assert(getIndexFileCount("default_nonindexmerge", "1") == 100)
sql("ALTER TABLE nonindexmerge COMPACT 'minor'").collect()
- val table = CarbonMetadata.getInstance().getCarbonTable("default","nonindexmerge")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "nonindexmerge")
new CarbonIndexFileMergeWriter(table)
.mergeCarbonIndexFilesOfSegment("0.1", table.getTablePath, false)
assert(getIndexFileCount("default_nonindexmerge", "0.1") == 0)
@@ -167,7 +170,7 @@ class CarbonIndexFileMergeTestCase
assert(getIndexFileCount("default_nonindexmerge", "2") == 100)
assert(getIndexFileCount("default_nonindexmerge", "3") == 100)
sql("ALTER TABLE nonindexmerge COMPACT 'minor'").collect()
- val table = CarbonMetadata.getInstance().getCarbonTable("default","nonindexmerge")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "nonindexmerge")
new CarbonIndexFileMergeWriter(table)
.mergeCarbonIndexFilesOfSegment("0.1", table.getTablePath, false)
assert(getIndexFileCount("default_nonindexmerge", "0") == 100)
@@ -190,18 +193,32 @@ class CarbonIndexFileMergeTestCase
sql("select * from mitable").show()
}
- private def getIndexFileCount(tableName: String, segment: String): Int = {
- val table = CarbonMetadata.getInstance().getCarbonTable(tableName)
- val path = CarbonTablePath
- .getSegmentPath(table.getAbsoluteTableIdentifier.getTablePath, segment)
- val carbonFiles = FileFactory.getCarbonFile(path).listFiles(new CarbonFileFilter {
- override def accept(file: CarbonFile): Boolean = file.getName.endsWith(CarbonTablePath
- .INDEX_FILE_EXT)
- })
- if (carbonFiles != null) {
- carbonFiles.length
+ private def getIndexFileCount(tableName: String, segmentNo: String): Int = {
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableName)
+ val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentNo)
+ if (FileFactory.isFileExist(segmentDir)) {
+ val indexFiles = new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir)
+ indexFiles.asScala.map { f =>
+ if (f._2 == null) {
+ 1
+ } else {
+ 0
+ }
+ }.sum
} else {
- 0
+ val segment = Segment.getSegment(segmentNo, carbonTable.getTablePath)
+ if (segment != null) {
+ val store = new SegmentFileStore(carbonTable.getTablePath, segment.getSegmentFileName)
+ store.getSegmentFile.getLocationMap.values().asScala.map { f =>
+ if (f.getMergeFileName == null) {
+ f.getFiles.size()
+ } else {
+ 0
+ }
+ }.sum
+ } else {
+ 0
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala
index 8f891ce..d49b962 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortFunctionTest.scala
@@ -22,9 +22,12 @@ import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.Segment
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
class CompactionSupportGlobalSortFunctionTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
val filePath: String = s"$resourcesPath/globalsort"
@@ -527,8 +530,12 @@ class CompactionSupportGlobalSortFunctionTest extends QueryTest with BeforeAndAf
private def getIndexFileCount(tableName: String, segmentNo: String = "0"): Int = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", tableName)
- val store = carbonTable.getAbsoluteTableIdentifier.getTablePath + "/Fact/Part0/Segment_" +
- segmentNo
- new SegmentIndexFileStore().getIndexFilesFromSegment(store).size()
+ val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentNo)
+ if (FileFactory.isFileExist(segmentDir)) {
+ new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir).size()
+ } else {
+ val segment = Segment.getSegment(segmentNo, carbonTable.getTablePath)
+ new SegmentFileStore(carbonTable.getTablePath, segment.getSegmentFileName).getIndexCarbonFiles.size()
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortParameterTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortParameterTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortParameterTest.scala
index 2da1ada..54c19c2 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortParameterTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortParameterTest.scala
@@ -23,8 +23,11 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+import org.apache.carbondata.core.datamap.Segment
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
+import org.apache.carbondata.core.util.path.CarbonTablePath
class CompactionSupportGlobalSortParameterTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
val filePath: String = s"$resourcesPath/globalsort"
@@ -567,8 +570,12 @@ class CompactionSupportGlobalSortParameterTest extends QueryTest with BeforeAndA
private def getIndexFileCount(tableName: String, segmentNo: String = "0"): Int = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", tableName)
- val store = carbonTable.getAbsoluteTableIdentifier.getTablePath + "/Fact/Part0/Segment_" +
- segmentNo
- new SegmentIndexFileStore().getIndexFilesFromSegment(store).size()
+ val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentNo)
+ if (FileFactory.isFileExist(segmentDir)) {
+ new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir).size()
+ } else {
+ val segment = Segment.getSegment(segmentNo, carbonTable.getTablePath)
+ new SegmentFileStore(carbonTable.getTablePath, segment.getSegmentFileName).getIndexCarbonFiles.size()
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
index f3e12d1..c695b05 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
@@ -26,8 +26,10 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.carbondata.core.datamap.Segment
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -189,12 +191,14 @@ class TestBatchSortDataLoad extends QueryTest with BeforeAndAfterAll {
}
def getIndexfileCount(tableName: String, segmentNo: String = "0"): Int = {
- val carbonTable = CarbonMetadata.getInstance().getCarbonTable(
- CarbonCommonConstants.DATABASE_DEFAULT_NAME,
- tableName
- )
- val segmentDir = carbonTable.getSegmentPath(segmentNo)
- new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir).size()
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", tableName)
+ val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentNo)
+ if (FileFactory.isFileExist(segmentDir)) {
+ new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir).size()
+ } else {
+ val segment = Segment.getSegment(segmentNo, carbonTable.getTablePath)
+ new SegmentFileStore(carbonTable.getTablePath, segment.getSegmentFileName).getIndexCarbonFiles.size()
+ }
}
override def afterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
index b9d8e12..39785a3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
@@ -17,6 +17,8 @@
package org.apache.carbondata.spark.testsuite.dataload
+import scala.collection.JavaConverters._
+
import java.io.{File, FilenameFilter}
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -26,7 +28,9 @@ import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.datamap.Segment
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
var originVersion = ""
@@ -49,12 +53,20 @@ class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
val indexReader = new CarbonIndexFileReader()
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "test_table_v3")
val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, "0")
- val carbonIndexPaths = new File(segmentDir)
- .listFiles(new FilenameFilter {
- override def accept(dir: File, name: String): Boolean = {
- name.endsWith(CarbonTablePath.getCarbonIndexExtension)
- }
- })
+
+ val carbonIndexPaths = if (FileFactory.isFileExist(segmentDir)) {
+ new File(segmentDir)
+ .listFiles(new FilenameFilter {
+ override def accept(dir: File, name: String): Boolean = {
+ name.endsWith(CarbonTablePath.getCarbonIndexExtension)
+ }
+ })
+ } else {
+ val segment = Segment.getSegment("0", carbonTable.getTablePath)
+ val store = new SegmentFileStore(carbonTable.getTablePath, segment.getSegmentFileName)
+ store.readIndexFiles()
+ store.getIndexCarbonFiles.asScala.map(f => new File(f.getAbsolutePath)).toArray
+ }
for (carbonIndexPath <- carbonIndexPaths) {
indexReader.openThriftReader(carbonIndexPath.getCanonicalPath)
assert(indexReader.readIndexHeader().getVersion === 3)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
index bba75ad..d7b1172 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
@@ -17,6 +17,8 @@
package org.apache.carbondata.spark.testsuite.dataload
+import scala.collection.JavaConverters._
+
import java.io.{File, FileWriter}
import org.apache.commons.io.FileUtils
@@ -31,8 +33,10 @@ import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+import org.apache.carbondata.core.datamap.Segment
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
import org.apache.carbondata.spark.rdd.CarbonScanRDD
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -273,7 +277,15 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
sql(s"LOAD DATA LOCAL INPATH '$inputPath' INTO TABLE carbon_globalsort")
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "carbon_globalsort")
val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, "0")
- assertResult(Math.max(7, defaultParallelism) + 1)(new File(segmentDir).listFiles().length)
+ if (FileFactory.isFileExist(segmentDir)) {
+ assertResult(Math.max(7, defaultParallelism) + 1)(new File(segmentDir).listFiles().length)
+ } else {
+ val segment = Segment.getSegment("0", carbonTable.getTablePath)
+ val store = new SegmentFileStore(carbonTable.getTablePath, segment.getSegmentFileName)
+ store.readIndexFiles()
+ val size = store.getIndexFilesMap.asScala.map(f => f._2.size()).sum
+ assertResult(Math.max(7, defaultParallelism) + 1)(size + store.getIndexFilesMap.size())
+ }
}
test("Query with small files") {
@@ -379,6 +391,11 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
private def getIndexFileCount(tableName: String, segmentNo: String = "0"): Int = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", tableName)
val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentNo)
- new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir).size()
+ if (FileFactory.isFileExist(segmentDir)) {
+ new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir).size()
+ } else {
+ val segment = Segment.getSegment(segmentNo, carbonTable.getTablePath)
+ new SegmentFileStore(carbonTable.getTablePath, segment.getSegmentFileName).getIndexCarbonFiles.size()
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
index b5c3df1..074c807 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
@@ -64,8 +64,9 @@ class CGDataMapFactory(
* Get the datamap for segmentid
*/
override def getDataMaps(segment: Segment): java.util.List[CoarseGrainDataMap] = {
- val path = CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo)
- val file = FileFactory.getCarbonFile(path+ "/" +dataMapSchema.getDataMapName)
+ val path = identifier.getTablePath
+ val file = FileFactory.getCarbonFile(
+ path+ "/" +dataMapSchema.getDataMapName + "/" + segment.getSegmentNo)
val files = file.listFiles()
files.map {f =>
@@ -100,8 +101,9 @@ class CGDataMapFactory(
* @return
*/
override def toDistributable(segment: Segment): java.util.List[DataMapDistributable] = {
- val path = CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo)
- val file = FileFactory.getCarbonFile(path+ "/" +dataMapSchema.getDataMapName)
+ val path = identifier.getTablePath
+ val file = FileFactory.getCarbonFile(
+ path+ "/" +dataMapSchema.getDataMapName + "/" + segment.getSegmentNo)
val files = file.listFiles()
files.map { f =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
index 2d666c3..08d8911 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -89,8 +89,9 @@ class FGDataMapFactory(carbonTable: CarbonTable,
* @return
*/
override def toDistributable(segment: Segment): java.util.List[DataMapDistributable] = {
- val path = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segment.getSegmentNo)
- val file = FileFactory.getCarbonFile(path+ "/" +dataMapSchema.getDataMapName)
+ val path = carbonTable.getTablePath
+ val file = FileFactory.getCarbonFile(
+ path+ "/" +dataMapSchema.getDataMapName + "/" + segment.getSegmentNo)
val files = file.listFiles()
files.map { f =>
@@ -416,7 +417,6 @@ class FGDataMapWriter(carbonTable: CarbonTable,
stream.write(bytes)
stream.writeInt(bytes.length)
stream.close()
-// commitFile(fgwritepath)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index eaa2ae7..642607c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -17,6 +17,8 @@
package org.apache.carbondata.spark.testsuite.datamap
+import scala.collection.JavaConverters._
+
import java.io.{File, FilenameFilter}
import org.apache.spark.sql.Row
@@ -26,7 +28,9 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.common.exceptions.MetadataProcessException
import org.apache.carbondata.common.exceptions.sql.{MalformedDataMapCommandException, NoSuchDataMapException}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.datamap.Segment
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -261,12 +265,21 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
| group by name
""".stripMargin)
assertResult(true)(new File(path).exists())
- assertResult(true)(new File(s"${CarbonTablePath.getSegmentPath(path, "0")}")
- .list(new FilenameFilter {
- override def accept(dir: File, name: String): Boolean = {
- name.contains(CarbonCommonConstants.FACT_FILE_EXT)
- }
- }).length > 0)
+ if (FileFactory.isFileExist(CarbonTablePath.getSegmentPath(path, "0"))) {
+ assertResult(true)(new File(s"${CarbonTablePath.getSegmentPath(path, "0")}")
+ .list(new FilenameFilter {
+ override def accept(dir: File, name: String): Boolean = {
+ name.contains(CarbonCommonConstants.FACT_FILE_EXT)
+ }
+ }).length > 0)
+ } else {
+ val segment = Segment.getSegment("0", path)
+ val store = new SegmentFileStore(path, segment.getSegmentFileName)
+ store.readIndexFiles()
+ val size = store.getIndexFilesMap.asScala.map(f => f._2.size()).sum
+ assertResult(true)(size > 0)
+ }
+
checkAnswer(sql("select name,avg(salary) from main group by name"), Row("amy", 13.0))
checkAnswer(sql("select * from main_preagg"), Row("amy", 26, 2))
sql("drop datamap preagg on table main")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala
new file mode 100644
index 0000000..d786d10
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.flatfolder
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class FlatFolderTableLoadingTestCase extends QueryTest with BeforeAndAfterAll {
+ override def beforeAll {
+ dropTable
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ sql(
+ """
+ | CREATE TABLE originTable (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ }
+
+ def validateDataFiles(tableUniqueName: String, segmentId: String): Unit = {
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
+ val files = FileFactory.getCarbonFile(carbonTable.getTablePath).listFiles()
+ assert(files.exists(_.getName.endsWith(CarbonTablePath.CARBON_DATA_EXT)))
+ }
+
+ test("data loading for flat folder with global sort") {
+ sql(
+ """
+ | CREATE TABLE flatfolder_gs (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int,empno int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('sort_scope'='global_sort', 'flat_folder'='true')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE flatfolder_gs OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ validateDataFiles("default_flatfolder_gs", "0")
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from flatfolder_gs order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+
+ }
+
+ test("data loading for flat folder") {
+ sql(
+ """
+ | CREATE TABLE flatfolder (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int,empno int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('flat_folder'='true')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE flatfolder OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ validateDataFiles("default_flatfolder", "0")
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from flatfolder order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+
+ }
+
+ test("data loading for flat folder pre-agg") {
+ sql(
+ """
+ | CREATE TABLE flatfolder_preagg (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int,empno int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('flat_folder'='true')
+ """.stripMargin)
+ sql("create datamap p2 on table flatfolder_preagg using 'preaggregate' as select empname, designation, min(salary) from flatfolder_preagg group by empname, designation ")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE flatfolder_preagg OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ validateDataFiles("default_flatfolder_preagg", "0")
+ validateDataFiles("default_flatfolder_preagg_p2", "0")
+
+ checkAnswer(sql("select empname, designation, min(salary) from flatfolder_preagg group by empname, designation"),
+ sql("select empname, designation, min(salary) from originTable group by empname, designation"))
+
+ }
+
+ override def afterAll = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION ,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT)
+ dropTable
+ }
+
+ def dropTable = {
+ sql("drop table if exists originTable")
+ sql("drop table if exists flatfolder")
+ sql("drop table if exists flatfolder_gs")
+ sql("drop table if exists flatfolder_preagg")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
index ec39f66..2432715 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
@@ -26,6 +26,9 @@ import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOp
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
class UpdateCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
@@ -689,7 +692,12 @@ class UpdateCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
CarbonCommonConstants.FILE_SEPARATOR + "t" +
CarbonCommonConstants.FILE_SEPARATOR + "Fact" +
CarbonCommonConstants.FILE_SEPARATOR + "Part0")
- assert(f.list().length == 2)
+ if (!FileFactory.isFileExist(
+ CarbonTablePath.getSegmentFilesLocation(
+ dblocation + CarbonCommonConstants.FILE_SEPARATOR +
+ CarbonCommonConstants.FILE_SEPARATOR + "t"))) {
+ assert(f.list().length == 2)
+ }
}
test("test sentences func in update statement") {
sql("drop table if exists senten")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
index 0eaaec5..133454a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
@@ -16,17 +16,22 @@
*/
package org.apache.carbondata.spark.testsuite.partition
+import scala.collection.JavaConverters._
+
import org.apache.spark.sql.Row
import org.apache.spark.sql.test.TestQueryExecutor
import org.scalatest.BeforeAndAfterAll
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.carbondata.core.datamap.Segment
+
class TestDataLoadingForPartitionTable extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
@@ -62,12 +67,20 @@ class TestDataLoadingForPartitionTable extends QueryTest with BeforeAndAfterAll
def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Seq[Int]): Unit = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
val segmentDir = carbonTable.getSegmentPath(segmentId)
- val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
- val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
- override def accept(file: CarbonFile): Boolean = {
- return file.getName.endsWith(".carbondata")
- }
- })
+
+ val dataFiles = if (FileFactory.isFileExist(segmentDir)) {
+ val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
+ carbonFile.listFiles(new CarbonFileFilter() {
+ override def accept(file: CarbonFile): Boolean = {
+ return file.getName.endsWith(".carbondata")
+ }
+ })
+ } else {
+ val segment = Segment.getSegment(segmentId, carbonTable.getTablePath)
+ val store = new SegmentFileStore(carbonTable.getTablePath, segment.getSegmentFileName)
+ store.readIndexFiles()
+ store.getIndexFilesMap.asScala.flatMap(_._2.asScala).map(f => FileFactory.getCarbonFile(f)).toArray
+ }
assert(dataFiles.size == partitions.size)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
index 0422239..f443214 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
@@ -41,7 +41,7 @@ import org.apache.carbondata.processing.sort.sortdata.SortParameters
import org.apache.carbondata.processing.store.{CarbonFactHandler, CarbonFactHandlerFactory}
import org.apache.carbondata.processing.util.{CarbonBadRecordUtil, CarbonDataProcessorUtil}
import org.apache.carbondata.spark.rdd.{NewRddIterator, StringArrayRow}
-import org.apache.carbondata.spark.util.Util
+import org.apache.carbondata.spark.util.{CarbonScalaUtil, Util}
object DataLoadProcessorStepOnSpark {
private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
index 2ba6e5e..3aaf0ae 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
@@ -28,6 +28,7 @@ import org.apache.spark.{Partition, SparkContext}
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.execution.command.CarbonMergerMapping
+import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.hadoop.{CarbonInputSplit, CarbonMultiBlockSplit}
import org.apache.carbondata.hadoop.api.{CarbonInputFormat, CarbonTableInputFormat}
@@ -74,7 +75,8 @@ class CarbonIUDMergerRDD[K, V](
val carbonInputSplits = splits.asScala.map(_.asInstanceOf[CarbonInputSplit])
// group blocks by segment.
- val splitsGroupedMySegment = carbonInputSplits.groupBy(_.getSegmentId)
+ val splitsGroupedMySegment =
+ carbonInputSplits.groupBy(_.getSegmentId)
var i = -1
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index d29284f..2fca57e 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -37,6 +37,7 @@ import org.apache.spark.sql.util.CarbonException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.datastore.block._
import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier}
@@ -133,7 +134,7 @@ class CarbonMergerRDD[K, V](
.toList
}
mergeNumber = if (CompactionType.IUD_UPDDEL_DELTA == carbonMergerMapping.campactionType) {
- tableBlockInfoList.get(0).getSegmentId
+ tableBlockInfoList.get(0).getSegment.toString
} else {
mergedLoadName.substring(
mergedLoadName.lastIndexOf(CarbonCommonConstants.LOAD_FOLDER) +
@@ -326,7 +327,9 @@ class CarbonMergerRDD[K, V](
val blockInfo = new TableBlockInfo(entry.getPath.toString,
entry.getStart, entry.getSegmentId,
entry.getLocations, entry.getLength, entry.getVersion,
- updateStatusManager.getDeleteDeltaFilePath(entry.getPath.toString, entry.getSegmentId)
+ updateStatusManager.getDeleteDeltaFilePath(
+ entry.getPath.toString,
+ Segment.toSegment(entry.getSegmentId).getSegmentNo)
)
(!updated || (updated && (!CarbonUtil
.isInvalidTableBlock(blockInfo.getSegmentId, blockInfo.getFilePath,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index 77ff139..3995aa7 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -530,6 +530,23 @@ object CommonUtil {
}
/**
+ * This method will validate the flat folder property specified by the user
+ *
+ * @param tableProperties
+ */
+ def validateFlatFolder(tableProperties: Map[String, String]): Unit = {
+ val tblPropName = CarbonCommonConstants.FLAT_FOLDER
+ if (tableProperties.get(tblPropName).isDefined) {
+ val trimStr = tableProperties(tblPropName).trim
+ if (!trimStr.equalsIgnoreCase("true") && !trimStr.equalsIgnoreCase("false")) {
+ throw new MalformedCarbonCommandException(s"Invalid $tblPropName value found: " +
+ s"$trimStr, only true|false is supported.")
+ }
+ tableProperties.put(tblPropName, trimStr)
+ }
+ }
+
+ /**
* This method will validate the compaction level threshold property specified by the user
* the property is used while doing minor compaction
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 61a5b42..7d28790 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -301,6 +301,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
CommonUtil.validateTableBlockSize(tableProperties)
// validate table level properties for compaction
CommonUtil.validateTableLevelCompactionProperties(tableProperties)
+ // validate flat folder property.
+ CommonUtil.validateFlatFolder(tableProperties)
TableModel(
ifNotExistPresent,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
index 84d9c47..fdbf400 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
@@ -25,18 +25,23 @@ import scala.collection.mutable
import scala.collection.mutable.ListBuffer
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapreduce.Job
import org.apache.spark.sql.execution.command.{AlterPartitionModel, DataMapField, Field, PartitionerField}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.datastore.block.{SegmentProperties, TableBlockInfo}
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier, SegmentFileStore}
import org.apache.carbondata.core.metadata.schema.PartitionInfo
import org.apache.carbondata.core.metadata.schema.partition.PartitionType
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.readcommitter.TableStatusReadCommittedScope
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.hadoop.CarbonInputSplit
@@ -192,9 +197,13 @@ object PartitionUtils {
val batchNo = CarbonTablePath.DataFileUtil.getBatchNoFromTaskNo(taskNo)
val taskId = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(taskNo)
val bucketNumber = CarbonTablePath.DataFileUtil.getBucketNo(path)
- val indexFilePath = CarbonTablePath.getCarbonIndexFilePath(
- tablePath, String.valueOf(taskId), segmentId, batchNo, String.valueOf(bucketNumber),
- timestamp, version)
+ val indexFilePath =
+ new Path(new Path(path).getParent,
+ CarbonTablePath.getCarbonIndexFileName(taskId,
+ bucketNumber.toInt,
+ batchNo,
+ timestamp,
+ segmentId)).toString
// indexFilePath could be duplicated when multiple data file related to one index file
if (indexFilePath != null && !pathList.contains(indexFilePath)) {
pathList.add(indexFilePath)
@@ -209,11 +218,13 @@ object PartitionUtils {
CarbonUtil.deleteFiles(files.asScala.toArray)
if (!files.isEmpty) {
val carbonTable = alterPartitionModel.carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
- val file = SegmentFileStore.writeSegmentFile(
- identifier.getTablePath,
- alterPartitionModel.segmentId,
- alterPartitionModel.carbonLoadModel.getFactTimeStamp.toString)
- val segmentFiles = Seq(new Segment(alterPartitionModel.segmentId, file, null))
+ val updatedSegFile: String = mergeAndUpdateSegmentFile(alterPartitionModel,
+ identifier,
+ segmentId,
+ carbonTable,
+ files.asScala)
+
+ val segmentFiles = Seq(new Segment(alterPartitionModel.segmentId, updatedSegFile, null))
.asJava
if (!CarbonUpdateUtil.updateTableMetadataStatus(
new util.HashSet[Segment](Seq(new Segment(alterPartitionModel.segmentId,
@@ -283,4 +294,50 @@ object PartitionUtils {
generatePartitionerField(allPartitionColumn.toList, Seq.empty)
}
+
+ private def mergeAndUpdateSegmentFile(alterPartitionModel: AlterPartitionModel,
+ identifier: AbsoluteTableIdentifier,
+ segmentId: String,
+ carbonTable: CarbonTable, filesToBeDelete: Seq[File]) = {
+ val metadataDetails =
+ SegmentStatusManager.readTableStatusFile(
+ CarbonTablePath.getTableStatusFilePath(carbonTable.getTablePath))
+ val segmentFile =
+ metadataDetails.find(_.getLoadName.equals(segmentId)).get.getSegmentFile
+ var allSegmentFiles: Seq[CarbonFile] = Seq.empty[CarbonFile]
+ val file = SegmentFileStore.writeSegmentFile(
+ carbonTable,
+ alterPartitionModel.segmentId,
+ System.currentTimeMillis().toString)
+ if (segmentFile != null) {
+ allSegmentFiles ++= FileFactory.getCarbonFile(
+ SegmentFileStore.getSegmentFilePath(carbonTable.getTablePath, segmentFile)) :: Nil
+ }
+ val updatedSegFile = {
+ val carbonFile = FileFactory.getCarbonFile(
+ SegmentFileStore.getSegmentFilePath(carbonTable.getTablePath, file))
+ allSegmentFiles ++= carbonFile :: Nil
+
+ val mergedSegFileName = SegmentFileStore.genSegmentFileName(
+ segmentId,
+ alterPartitionModel.carbonLoadModel.getFactTimeStamp.toString)
+ val tmpFile = mergedSegFileName + "_tmp"
+ val segmentStoreFile = SegmentFileStore.mergeSegmentFiles(
+ tmpFile,
+ CarbonTablePath.getSegmentFilesLocation(carbonTable.getTablePath),
+ allSegmentFiles.toArray)
+ val indexFiles = segmentStoreFile.getLocationMap.values().asScala.head.getFiles
+ filesToBeDelete.foreach(f => indexFiles.remove(f.getName))
+ SegmentFileStore.writeSegmentFile(
+ segmentStoreFile,
+ CarbonTablePath.getSegmentFilesLocation(carbonTable.getTablePath) +
+ CarbonCommonConstants.FILE_SEPARATOR + mergedSegFileName + CarbonTablePath.SEGMENT_EXT)
+ carbonFile.delete()
+ FileFactory.getCarbonFile(
+ SegmentFileStore.getSegmentFilePath(
+ carbonTable.getTablePath, tmpFile + CarbonTablePath.SEGMENT_EXT)).delete()
+ mergedSegFileName + CarbonTablePath.SEGMENT_EXT
+ }
+ updatedSegFile
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
index 5902783..f3f2650 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
@@ -84,9 +84,7 @@ object IndexDataMapRebuildRDD {
segmentId: String): Unit = {
val dataMapStorePath =
- CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentId) +
- File.separator +
- dataMapName
+ CarbonTablePath.getDataMapStorePath(carbonTable.getTablePath, segmentId, dataMapName)
if (!FileFactory.isFileExist(dataMapStorePath)) {
if (FileFactory.mkdirs(dataMapStorePath, FileFactory.getFileType(dataMapStorePath))) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 21a8641..5d53ccc 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -50,6 +50,8 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.datamap.status.DataMapStatusManager
import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.dictionary.server.DictionaryServer
import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnarFormatVersion, SegmentFileStore}
@@ -434,13 +436,7 @@ object CarbonDataRDDFactory {
segmentDetails.add(new Segment(resultOfBlock._2._1.getLoadName, null))
}
}
- val segmentFiles = segmentDetails.asScala.map{seg =>
- val file = SegmentFileStore.writeSegmentFile(
- carbonTable.getTablePath,
- seg.getSegmentNo,
- updateModel.get.updatedTimeStamp.toString)
- new Segment(seg.getSegmentNo, file)
- }.filter(_.getSegmentFileName != null).asJava
+ val segmentFiles = updateSegmentFiles(carbonTable, segmentDetails, updateModel.get)
// this means that the update doesnt have any records to update so no need to do table
// status file updation.
@@ -517,9 +513,13 @@ object CarbonDataRDDFactory {
writeDictionary(carbonLoadModel, result, writeAll = false)
val segmentFileName =
- SegmentFileStore.writeSegmentFile(carbonTable.getTablePath, carbonLoadModel.getSegmentId,
+ SegmentFileStore.writeSegmentFile(carbonTable, carbonLoadModel.getSegmentId,
String.valueOf(carbonLoadModel.getFactTimeStamp))
+ SegmentFileStore.updateSegmentFile(
+ carbonTable.getTablePath,
+ carbonLoadModel.getSegmentId,
+ segmentFileName)
operationContext.setProperty(carbonTable.getTableUniqueName + "_Segment",
carbonLoadModel.getSegmentId)
val loadTablePreStatusUpdateEvent: LoadTablePreStatusUpdateEvent =
@@ -588,6 +588,58 @@ object CarbonDataRDDFactory {
}
/**
+ * Add and update the segment files. In case of update scenario the carbonindex files are written
+ * to the same segment so we need to update old segment file. So this ethod writes the latest data
+ * to new segment file and merges this file old file to get latest updated files.
+ * @param carbonTable
+ * @param segmentDetails
+ * @return
+ */
+ private def updateSegmentFiles(
+ carbonTable: CarbonTable,
+ segmentDetails: util.HashSet[Segment],
+ updateModel: UpdateTableModel) = {
+ val metadataDetails =
+ SegmentStatusManager.readTableStatusFile(
+ CarbonTablePath.getTableStatusFilePath(carbonTable.getTablePath))
+ val segmentFiles = segmentDetails.asScala.map { seg =>
+ val segmentFile =
+ metadataDetails.find(_.getLoadName.equals(seg.getSegmentNo)).get.getSegmentFile
+ var segmentFiles: Seq[CarbonFile] = Seq.empty[CarbonFile]
+
+ val file = SegmentFileStore.writeSegmentFile(
+ carbonTable,
+ seg.getSegmentNo,
+ String.valueOf(System.currentTimeMillis()))
+
+ if (segmentFile != null) {
+ segmentFiles ++= FileFactory.getCarbonFile(
+ SegmentFileStore.getSegmentFilePath(carbonTable.getTablePath, segmentFile)) :: Nil
+ }
+ val updatedSegFile = if (file != null) {
+ val carbonFile = FileFactory.getCarbonFile(
+ SegmentFileStore.getSegmentFilePath(carbonTable.getTablePath, file))
+ segmentFiles ++= carbonFile :: Nil
+
+ val mergedSegFileName = SegmentFileStore.genSegmentFileName(
+ seg.getSegmentNo,
+ updateModel.updatedTimeStamp.toString)
+ SegmentFileStore.mergeSegmentFiles(
+ mergedSegFileName,
+ CarbonTablePath.getSegmentFilesLocation(carbonTable.getTablePath),
+ segmentFiles.toArray)
+ carbonFile.delete()
+ mergedSegFileName + CarbonTablePath.SEGMENT_EXT
+ } else {
+ null
+ }
+
+ new Segment(seg.getSegmentNo, updatedSegFile)
+ }.filter(_.getSegmentFileName != null).asJava
+ segmentFiles
+ }
+
+ /**
* If data load is triggered by UPDATE query, this func will execute the update
* TODO: move it to a separate update command
*/
@@ -614,10 +666,11 @@ object CarbonDataRDDFactory {
carbonTable.getMetadataPath)
.filter(lmd => lmd.getSegmentStatus.equals(SegmentStatus.LOAD_PARTIAL_SUCCESS) ||
lmd.getSegmentStatus.equals(SegmentStatus.SUCCESS))
- val segmentIds = loadMetadataDetails.map(_.getLoadName)
- val segmentIdIndex = segmentIds.zipWithIndex.toMap
- val segmentId2maxTaskNo = segmentIds.map { segId =>
- (segId, CarbonUpdateUtil.getLatestTaskIdForSegment(segId, carbonLoadModel.getTablePath))
+ val segments = loadMetadataDetails.map(f => new Segment(f.getLoadName, f.getSegmentFile))
+ val segmentIdIndex = segments.map(_.getSegmentNo).zipWithIndex.toMap
+ val segmentId2maxTaskNo = segments.map { seg =>
+ (seg.getSegmentNo,
+ CarbonUpdateUtil.getLatestTaskIdForSegment(seg, carbonLoadModel.getTablePath))
}.toMap
class SegmentPartitioner(segIdIndex: Map[String, Int], parallelism: Int)
@@ -639,10 +692,14 @@ object CarbonDataRDDFactory {
val partitionId = TaskContext.getPartitionId()
val segIdIndex = partitionId / segmentUpdateParallelism
val randomPart = partitionId - segIdIndex * segmentUpdateParallelism
- val segId = segmentIds(segIdIndex)
- val newTaskNo = segmentId2maxTaskNo(segId) + randomPart + 1
- List(triggerDataLoadForSegment(carbonLoadModel, updateModel, segId, newTaskNo, partition)
- .toList).toIterator
+ val segId = segments(segIdIndex)
+ val newTaskNo = segmentId2maxTaskNo(segId.getSegmentNo) + randomPart + 1
+ List(triggerDataLoadForSegment(
+ carbonLoadModel,
+ updateModel,
+ segId.getSegmentNo,
+ newTaskNo,
+ partition).toList).toIterator
}.collect()
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
index 155bdd1..7605b9d 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
@@ -223,7 +223,7 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
if (compactionType == CompactionType.IUD_UPDDEL_DELTA) {
val segmentFilesList = loadsToMerge.asScala.map{seg =>
val file = SegmentFileStore.writeSegmentFile(
- carbonTable.getTablePath,
+ carbonTable,
seg.getLoadName,
carbonLoadModel.getFactTimeStamp.toString)
new Segment(seg.getLoadName, file)
@@ -231,7 +231,7 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
segmentFilesForIUDCompact = new util.ArrayList[Segment](segmentFilesList)
} else {
segmentFileName = SegmentFileStore.writeSegmentFile(
- carbonTable.getTablePath,
+ carbonTable,
mergedLoadNumber,
carbonLoadModel.getFactTimeStamp.toString)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index 93c0b4a..30cb464 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -99,7 +99,7 @@ class CarbonSession(@transient val sc: SparkContext,
trySearchMode(qe, sse)
} catch {
case e: Exception =>
- logError(String.format(
+ log.error(String.format(
"Exception when executing search mode: %s", e.getMessage))
throw e;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
index 0c6d2ba..127e1b1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
@@ -40,7 +40,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, DeleteDeltaBlockDetails, SegmentUpdateDetails, TupleIdEnum}
import org.apache.carbondata.core.mutate.data.RowCountDetailsVO
-import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentUpdateStatusManager}
+import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager, SegmentUpdateStatusManager}
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl
import org.apache.carbondata.hadoop.api.{CarbonInputFormat, CarbonTableInputFormat}
@@ -68,12 +68,7 @@ object DeleteExecution {
val database = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession)
val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession)
val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
- val isPartitionTable = carbonTable.isHivePartitionTable
- val factPath = if (isPartitionTable) {
- absoluteTableIdentifier.getTablePath
- } else {
- CarbonTablePath.getFactDir(absoluteTableIdentifier.getTablePath)
- }
+ val tablePath = absoluteTableIdentifier.getTablePath
var segmentsTobeDeleted = Seq.empty[Segment]
val deleteRdd = if (isUpdateOperation) {
@@ -114,6 +109,9 @@ object DeleteExecution {
CarbonUpdateUtil
.createBlockDetailsMap(blockMappingVO, segmentUpdateStatusMngr)
+ val metadataDetails = SegmentStatusManager.readTableStatusFile(
+ CarbonTablePath.getTableStatusFilePath(carbonTable.getTablePath))
+
val rowContRdd =
sparkSession.sparkContext.parallelize(
blockMappingVO.getCompleteBlockRowDetailVO.asScala.toSeq,
@@ -127,12 +125,16 @@ object DeleteExecution {
var result = List[(SegmentStatus, (SegmentUpdateDetails, ExecutionErrors))]()
while (records.hasNext) {
val ((key), (rowCountDetailsVO, groupedRows)) = records.next
+ val segmentId = key.substring(0, key.indexOf(CarbonCommonConstants.FILE_SEPARATOR))
+ val segmentFile =
+ metadataDetails.find(_.getLoadName.equals(segmentId)).get.getSegmentFile
result = result ++
deleteDeltaFunc(index,
key,
groupedRows.toIterator,
timestamp,
- rowCountDetailsVO)
+ rowCountDetailsVO,
+ segmentFile)
}
result
}
@@ -219,7 +221,8 @@ object DeleteExecution {
key: String,
iter: Iterator[Row],
timestamp: String,
- rowCountDetailsVO: RowCountDetailsVO
+ rowCountDetailsVO: RowCountDetailsVO,
+ segmentFile: String
): Iterator[(SegmentStatus, (SegmentUpdateDetails, ExecutionErrors))] = {
val result = new DeleteDelataResultImpl()
@@ -255,7 +258,7 @@ object DeleteExecution {
countOfRows = countOfRows + 1
}
- val blockPath = CarbonUpdateUtil.getTableBlockPath(TID, factPath, isPartitionTable)
+ val blockPath = CarbonUpdateUtil.getTableBlockPath(TID, tablePath, segmentFile != null)
val completeBlockName = CarbonTablePath
.addDataPartPrefix(CarbonUpdateUtil.getRequiredFieldFromTID(TID, TupleIdEnum.BLOCK_ID) +
CarbonCommonConstants.FACT_FILE_EXT)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
index f3b4be7..857cd81 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
@@ -103,6 +103,9 @@ case class PreAggregateTableHelper(
.LOAD_SORT_SCOPE_DEFAULT))
tableProperties
.put(CarbonCommonConstants.TABLE_BLOCKSIZE, parentTable.getBlockSizeInMB.toString)
+ tableProperties.put(CarbonCommonConstants.FLAT_FOLDER,
+ parentTable.getTableInfo.getFactTable.getTableProperties.asScala.getOrElse(
+ CarbonCommonConstants.FLAT_FOLDER, CarbonCommonConstants.DEFAULT_FLAT_FOLDER))
val tableIdentifier =
TableIdentifier(parentTable.getTableName + "_" + dataMapName,
Some(parentTable.getDatabaseName))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
index 7d15cc1..617f5e8 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
@@ -138,6 +138,11 @@ private[sql] case class CarbonDescribeFormattedCommand(
tblProps.get(CarbonCommonConstants.TABLE_ALLOWED_COMPACTION_DAYS),
CarbonCommonConstants.DEFAULT_DAYS_ALLOWED_TO_COMPACT))
}
+ if (tblProps.containsKey(CarbonCommonConstants.FLAT_FOLDER)) {
+ results ++= Seq((CarbonCommonConstants.FLAT_FOLDER.toUpperCase,
+ tblProps.get(CarbonCommonConstants.FLAT_FOLDER),
+ CarbonCommonConstants.DEFAULT_FLAT_FOLDER))
+ }
results ++= Seq(("", "", ""), ("##Detailed Column property", "", ""))
if (colPropStr.length() > 0) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
index 0bdef8a..7cee409 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
@@ -20,13 +20,15 @@ package org.apache.carbondata.spark.testsuite.partition
import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
+import org.apache.hadoop.fs.Path
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -855,15 +857,24 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
validatePartitionTableFiles(partitions, dataFiles)
}
- def getDataFiles(carbonTable: CarbonTable, segmentId: String): Array[CarbonFile] = {
- val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentId)
- val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
- val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
- override def accept(file: CarbonFile): Boolean = {
- return file.getName.endsWith(".carbondata")
- }
- })
- dataFiles
+ def getDataFiles(carbonTable: CarbonTable, segmentId: String): Array[String] = {
+ val segment = Segment.getSegment(segmentId, carbonTable.getTablePath)
+ if (segment.getSegmentFileName != null) {
+ val sfs = new SegmentFileStore(carbonTable.getTablePath, segment.getSegmentFileName)
+ sfs.readIndexFiles()
+ val indexFilesMap = sfs.getIndexFilesMap
+ val dataFiles = indexFilesMap.asScala.flatMap(_._2.asScala).map(f => new Path(f).getName)
+ dataFiles.toArray
+ } else {
+ val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentId)
+ val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
+ val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ override def accept(file: CarbonFile): Boolean = {
+ return file.getName.endsWith(".carbondata")
+ }
+ })
+ dataFiles.map(_.getName)
+ }
}
/**
@@ -871,10 +882,10 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
* @param partitions
* @param dataFiles
*/
- def validatePartitionTableFiles(partitions: Seq[Int], dataFiles: Array[CarbonFile]): Unit = {
+ def validatePartitionTableFiles(partitions: Seq[Int], dataFiles: Array[String]): Unit = {
val partitionIds: ListBuffer[Int] = new ListBuffer[Int]()
dataFiles.foreach { dataFile =>
- val partitionId = CarbonTablePath.DataFileUtil.getTaskNo(dataFile.getName).split("_")(0).toInt
+ val partitionId = CarbonTablePath.DataFileUtil.getTaskNo(dataFile).split("_")(0).toInt
partitionIds += partitionId
assert(partitions.contains(partitionId))
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala
index 48733dc..1c7cb10 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala
@@ -43,9 +43,9 @@ class CarbonGetTableDetailCommandTestCase extends QueryTest with BeforeAndAfterA
assertResult(2)(result.length)
assertResult("table_info1")(result(0).getString(0))
// 2096 is the size of carbon table
- assertResult(2096)(result(0).getLong(1))
+ assertResult(2098)(result(0).getLong(1))
assertResult("table_info2")(result(1).getString(0))
- assertResult(2096)(result(1).getLong(1))
+ assertResult(2098)(result(1).getLong(1))
}
override def afterAll: Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index 3dc34d3..bfa498e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -165,6 +165,7 @@ public class DataMapWriterListener {
writer.finish();
}
}
+ registry.clear();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java b/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java
index eb02ede..fcabef5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java
@@ -170,7 +170,8 @@ public abstract class AbstractDataLoadProcessorStep {
carbonDataFileAttributes.getTaskId(),
bucketId,
0,
- String.valueOf(carbonDataFileAttributes.getFactTimeStamp())));
+ String.valueOf(carbonDataFileAttributes.getFactTimeStamp()),
+ configuration.getSegmentId()));
return listener;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java b/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
index bc28ace..5bed8b1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
@@ -64,7 +64,7 @@ public class TableProcessingOperations {
CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
@Override public boolean accept(CarbonFile path) {
String segmentId =
- CarbonTablePath.DataFileUtil.getSegmentId(path.getAbsolutePath() + "/dummy");
+ CarbonTablePath.DataFileUtil.getSegmentIdFromPath(path.getAbsolutePath() + "/dummy");
boolean found = false;
for (int j = 0; j < details.length; j++) {
if (details[j].getLoadName().equals(segmentId)) {
@@ -76,8 +76,8 @@ public class TableProcessingOperations {
}
});
for (int k = 0; k < listFiles.length; k++) {
- String segmentId =
- CarbonTablePath.DataFileUtil.getSegmentId(listFiles[k].getAbsolutePath() + "/dummy");
+ String segmentId = CarbonTablePath.DataFileUtil
+ .getSegmentIdFromPath(listFiles[k].getAbsolutePath() + "/dummy");
if (isCompactionFlow) {
if (segmentId.contains(".")) {
CarbonLoaderUtil.deleteStorePath(listFiles[k].getAbsolutePath());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index 90c297e..4d3f3fc 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.dictionary.service.DictionaryServiceProvider;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
@@ -83,7 +84,7 @@ public class CarbonLoadModel implements Serializable {
/**
* load Id
*/
- private String segmentId;
+ private Segment segment;
private String allDictPath;
@@ -424,7 +425,7 @@ public class CarbonLoadModel implements Serializable {
copy.blocksID = blocksID;
copy.taskNo = taskNo;
copy.factTimeStamp = factTimeStamp;
- copy.segmentId = segmentId;
+ copy.segment = segment;
copy.serializationNullFormat = serializationNullFormat;
copy.badRecordsLoggerEnable = badRecordsLoggerEnable;
copy.badRecordsAction = badRecordsAction;
@@ -479,7 +480,7 @@ public class CarbonLoadModel implements Serializable {
copyObj.blocksID = blocksID;
copyObj.taskNo = taskNo;
copyObj.factTimeStamp = factTimeStamp;
- copyObj.segmentId = segmentId;
+ copyObj.segment = segment;
copyObj.serializationNullFormat = serializationNullFormat;
copyObj.badRecordsLoggerEnable = badRecordsLoggerEnable;
copyObj.badRecordsAction = badRecordsAction;
@@ -609,14 +610,24 @@ public class CarbonLoadModel implements Serializable {
* @return load Id
*/
public String getSegmentId() {
- return segmentId;
+ if (segment != null) {
+ return segment.getSegmentNo();
+ } else {
+ return null;
+ }
}
/**
* @param segmentId
*/
public void setSegmentId(String segmentId) {
- this.segmentId = segmentId;
+ if (segmentId != null) {
+ this.segment = Segment.toSegment(segmentId);
+ }
+ }
+
+ public Segment getSegment() {
+ return segment;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
index 7a11c8b..b22599d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.processing.merger;
+import java.io.IOException;
import java.util.List;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
@@ -46,10 +47,11 @@ public abstract class AbstractResultProcessor {
public abstract void close();
protected void setDataFileAttributesInModel(CarbonLoadModel loadModel,
- CompactionType compactionType, CarbonFactDataHandlerModel carbonFactDataHandlerModel) {
+ CompactionType compactionType, CarbonFactDataHandlerModel carbonFactDataHandlerModel)
+ throws IOException {
CarbonDataFileAttributes carbonDataFileAttributes;
if (compactionType == CompactionType.IUD_UPDDEL_DELTA) {
- long taskNo = CarbonUpdateUtil.getLatestTaskIdForSegment(loadModel.getSegmentId(),
+ long taskNo = CarbonUpdateUtil.getLatestTaskIdForSegment(loadModel.getSegment(),
loadModel.getTablePath());
// Increase the Task Index as in IUD_UPDDEL_DELTA_COMPACTION the new file will
// be written in same segment. So the TaskNo should be incremented by 1 from max val.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index fef8ab9..dde18a9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -175,7 +175,6 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
partitionSpec.getLocation().toString(), carbonLoadModel.getFactTimeStamp() + "",
partitionSpec.getPartitions());
} catch (IOException e) {
- isCompactionSuccess = false;
throw e;
}
}
@@ -428,6 +427,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel
.getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName,
tempStoreLocation, carbonStoreLocation);
+ carbonFactDataHandlerModel.setSegmentId(carbonLoadModel.getSegmentId());
setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonFactDataHandlerModel);
dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel,
CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
index 9a3258e..b877d52 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
@@ -63,7 +63,8 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
public RowResultMergerProcessor(String databaseName,
String tableName, SegmentProperties segProp, String[] tempStoreLocation,
- CarbonLoadModel loadModel, CompactionType compactionType, PartitionSpec partitionSpec) {
+ CarbonLoadModel loadModel, CompactionType compactionType, PartitionSpec partitionSpec)
+ throws IOException {
this.segprop = segProp;
this.partitionSpec = partitionSpec;
this.loadModel = loadModel;
@@ -84,6 +85,7 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
tempStoreLocation, carbonStoreLocation);
setDataFileAttributesInModel(loadModel, compactionType, carbonFactDataHandlerModel);
carbonFactDataHandlerModel.setCompactionFlow(true);
+ carbonFactDataHandlerModel.setSegmentId(loadModel.getSegmentId());
dataHandler = new CarbonFactDataHandlerColumnar(carbonFactDataHandlerModel);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
index 221697f..9b09269 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
@@ -59,6 +59,7 @@ public class RowResultProcessor {
carbonFactDataHandlerModel.setBucketId(bucketId);
//Note: set compaction flow just to convert decimal type
carbonFactDataHandlerModel.setCompactionFlow(true);
+ carbonFactDataHandlerModel.setSegmentId(loadModel.getSegmentId());
dataHandler = new CarbonFactDataHandlerColumnar(carbonFactDataHandlerModel);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 87a6de0..27249ab 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -267,7 +267,8 @@ public class CarbonFactDataHandlerModel {
carbonDataFileAttributes.getTaskId(),
bucketId,
0,
- String.valueOf(carbonDataFileAttributes.getFactTimeStamp())));
+ String.valueOf(carbonDataFileAttributes.getFactTimeStamp()),
+ configuration.getSegmentId()));
}
carbonFactDataHandlerModel.dataMapWriterlistener = listener;
carbonFactDataHandlerModel.writingCoresCount = configuration.getWritingCoresCount();
@@ -337,7 +338,8 @@ public class CarbonFactDataHandlerModel {
CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(loadModel.getTaskNo()),
carbonFactDataHandlerModel.getBucketId(),
carbonFactDataHandlerModel.getTaskExtension(),
- String.valueOf(loadModel.getFactTimeStamp())));
+ String.valueOf(loadModel.getFactTimeStamp()),
+ loadModel.getSegmentId()));
carbonFactDataHandlerModel.dataMapWriterlistener = listener;
return carbonFactDataHandlerModel;
[30/50] [abbrv] carbondata git commit: [CARBONDATA-2428] Support flat
folder for managed carbon table
Posted by ja...@apache.org.
[CARBONDATA-2428] Support flat folder for managed carbon table
Currently carbondata writing happens in fixed path tablepath/Fact/Part0/Segment_NUM folder and it is not same as hive/parquet folder structure. This PR makes all files written will be inside tablepath, it does not maintain any segment folder structure. Only for partition it adds the folder.
This feature can be controlled through a table property flat_folder. The default value of it is false.
This closes #2207
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/60dfdd38
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/60dfdd38
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/60dfdd38
Branch: refs/heads/carbonstore
Commit: 60dfdd3857d037231844d9fb95967dcdb0071f40
Parents: 181f0ac
Author: ravipesala <ra...@gmail.com>
Authored: Fri Apr 20 07:34:00 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Jun 16 10:56:10 2018 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 8 ++
.../apache/carbondata/core/datamap/Segment.java | 20 +++
.../carbondata/core/datamap/TableDataMap.java | 4 +-
.../core/datamap/dev/DataMapWriter.java | 3 +-
.../core/datastore/block/TableBlockInfo.java | 27 ++--
.../blockletindex/BlockletDataMapFactory.java | 4 +-
.../core/metadata/SegmentFileStore.java | 91 ++++++++++----
.../core/metadata/schema/table/CarbonTable.java | 13 ++
.../core/mutate/CarbonUpdateUtil.java | 71 ++++++-----
.../executor/impl/AbstractQueryExecutor.java | 18 ++-
.../core/scan/result/BlockletScannedResult.java | 3 +-
.../scan/scanner/impl/BlockletFullScanner.java | 6 +-
.../SegmentUpdateStatusManager.java | 40 +++---
.../apache/carbondata/core/util/CarbonUtil.java | 18 +--
.../core/util/path/CarbonTablePath.java | 77 +++++++++---
.../datastore/block/TableBlockInfoTest.java | 2 +-
.../CarbonFormatDirectoryStructureTest.java | 22 +---
.../bloom/BloomCoarseGrainDataMapFactory.java | 10 +-
.../lucene/LuceneDataMapFactoryBase.java | 17 ++-
.../carbondata/hadoop/CarbonInputSplit.java | 35 ++++--
.../hadoop/api/CarbonInputFormat.java | 2 +-
.../hadoop/api/CarbonTableInputFormat.java | 12 +-
.../sdv/generated/DataLoadingIUDTestCase.scala | 7 +-
.../sdv/generated/MergeIndexTestCase.scala | 55 ++++----
.../lucene/LuceneFineGrainDataMapSuite.scala | 36 ++++--
.../dataload/TestLoadDataGeneral.scala | 9 +-
.../InsertIntoCarbonTableTestCase.scala | 33 ++---
.../CarbonIndexFileMergeTestCase.scala | 53 +++++---
...ompactionSupportGlobalSortFunctionTest.scala | 15 ++-
...mpactionSupportGlobalSortParameterTest.scala | 15 ++-
.../dataload/TestBatchSortDataLoad.scala | 18 +--
.../dataload/TestDataLoadWithFileName.scala | 26 ++--
.../dataload/TestGlobalSortDataLoad.scala | 23 +++-
.../testsuite/datamap/CGDataMapTestCase.scala | 10 +-
.../testsuite/datamap/FGDataMapTestCase.scala | 6 +-
.../testsuite/datamap/TestDataMapCommand.scala | 27 ++--
.../FlatFolderTableLoadingTestCase.scala | 125 +++++++++++++++++++
.../iud/UpdateCarbonTableTestCase.scala | 10 +-
.../TestDataLoadingForPartitionTable.scala | 27 ++--
.../load/DataLoadProcessorStepOnSpark.scala | 2 +-
.../spark/rdd/CarbonIUDMergerRDD.scala | 4 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 7 +-
.../carbondata/spark/util/CommonUtil.scala | 17 +++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +
.../org/apache/spark/util/PartitionUtils.scala | 73 +++++++++--
.../datamap/IndexDataMapRebuildRDD.scala | 4 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 89 ++++++++++---
.../spark/rdd/CarbonTableCompactor.scala | 4 +-
.../org/apache/spark/sql/CarbonSession.scala | 2 +-
.../command/mutation/DeleteExecution.scala | 23 ++--
.../preaaggregate/PreAggregateTableHelper.scala | 3 +
.../table/CarbonDescribeFormattedCommand.scala | 5 +
.../partition/TestAlterPartitionTable.scala | 35 ++++--
.../CarbonGetTableDetailComandTestCase.scala | 4 +-
.../datamap/DataMapWriterListener.java | 1 +
.../loading/AbstractDataLoadProcessorStep.java | 3 +-
.../loading/TableProcessingOperations.java | 6 +-
.../loading/model/CarbonLoadModel.java | 21 +++-
.../merger/AbstractResultProcessor.java | 6 +-
.../merger/CompactionResultSortProcessor.java | 2 +-
.../merger/RowResultMergerProcessor.java | 4 +-
.../partition/spliter/RowResultProcessor.java | 1 +
.../store/CarbonFactDataHandlerModel.java | 6 +-
.../store/writer/AbstractFactDataWriter.java | 6 +-
.../processing/util/CarbonLoaderUtil.java | 4 +-
.../store/worker/SearchRequestHandler.java | 16 ++-
.../streaming/CarbonStreamRecordWriter.java | 2 +-
67 files changed, 961 insertions(+), 389 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 19ff494..2fcf0f5 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -718,6 +718,11 @@ public final class CarbonCommonConstants {
public static final String DEFAULT_ENABLE_AUTO_LOAD_MERGE = "false";
/**
+ * DEFAULT_FLAT_FOLDER
+ */
+ public static final String DEFAULT_FLAT_FOLDER = "false";
+
+ /**
* ZOOKEEPER_ENABLE_LOCK if this is set to true then zookeeper will be used to handle locking
* mechanism of carbon
*/
@@ -929,6 +934,9 @@ public final class CarbonCommonConstants {
public static final String TABLE_COMPACTION_PRESERVE_SEGMENTS = "compaction_preserve_segments";
// table property name of allowed compaction days while compaction
public static final String TABLE_ALLOWED_COMPACTION_DAYS = "allowed_compaction_days";
+ // Flat folder support on table. when it is true all carbondata files store directly under table
+ // path instead of sub folders.
+ public static final String FLAT_FOLDER = "flat_folder";
/**
* 16 mb size
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/datamap/Segment.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/Segment.java b/core/src/main/java/org/apache/carbondata/core/datamap/Segment.java
index 7b63b84..425cdf6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/Segment.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/Segment.java
@@ -162,6 +162,16 @@ public class Segment implements Serializable {
}
/**
+ * Converts to segment object
+ * @param segmentId
+ * @return
+ */
+ public static Segment toSegment(String segmentId) {
+ // SegmentId can be combination of segmentNo and segmentFileName.
+ return toSegment(segmentId, null);
+ }
+
+ /**
* Read the table status and get the segment corresponding to segmentNo
* @param segmentNo
* @param tablePath
@@ -170,6 +180,16 @@ public class Segment implements Serializable {
public static Segment getSegment(String segmentNo, String tablePath) {
LoadMetadataDetails[] loadMetadataDetails =
SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(tablePath));
+ return getSegment(segmentNo, loadMetadataDetails);
+ }
+
+ /**
+ * Get the segment object corresponding to segmentNo
+ * @param segmentNo
+ * @param loadMetadataDetails
+ * @return
+ */
+ public static Segment getSegment(String segmentNo, LoadMetadataDetails[] loadMetadataDetails) {
for (LoadMetadataDetails details: loadMetadataDetails) {
if (details.getLoadName().equals(segmentNo)) {
return new Segment(details.getLoadName(), details.getSegmentFile());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index 4ce0f6c..cb7ec03 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -103,7 +103,7 @@ public final class TableDataMap extends OperationEventListener {
}
blocklets.addAll(addSegmentId(
blockletDetailsFetcher.getExtendedBlocklets(pruneBlocklets, segment),
- segment.getSegmentNo()));
+ segment.toString()));
}
return blocklets;
}
@@ -182,7 +182,7 @@ public final class TableDataMap extends OperationEventListener {
detailedBlocklet.setDataMapWriterPath(blockletwritePath);
serializer.serializeBlocklet((FineGrainBlocklet) blocklet, blockletwritePath);
}
- detailedBlocklet.setSegmentId(distributable.getSegment().getSegmentNo());
+ detailedBlocklet.setSegmentId(distributable.getSegment().toString());
detailedBlocklets.add(detailedBlocklet);
}
return detailedBlocklets;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
index 89d5d76..8c8d2d8 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
@@ -16,7 +16,6 @@
*/
package org.apache.carbondata.core.datamap.dev;
-import java.io.File;
import java.io.IOException;
import java.util.List;
@@ -133,7 +132,7 @@ public abstract class DataMapWriter {
*/
public static String getDefaultDataMapPath(
String tablePath, String segmentId, String dataMapName) {
- return CarbonTablePath.getSegmentPath(tablePath, segmentId) + File.separator + dataMapName;
+ return CarbonTablePath.getDataMapStorePath(tablePath, segmentId, dataMapName);
}
public boolean isWritingFinished() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index c0cebe0..34d406b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
@@ -64,7 +65,7 @@ public class TableBlockInfo implements Distributable, Serializable {
/**
* id of the segment this will be used to sort the blocks
*/
- private String segmentId;
+ private Segment segment;
/**
* id of the Blocklet.
@@ -120,7 +121,7 @@ public class TableBlockInfo implements Distributable, Serializable {
this.filePath = FileFactory.getUpdatedFilePath(filePath);
this.blockletId = "0";
this.blockOffset = blockOffset;
- this.segmentId = segmentId;
+ this.segment = Segment.toSegment(segmentId);
this.locations = locations;
this.blockLength = blockLength;
this.version = version;
@@ -196,7 +197,7 @@ public class TableBlockInfo implements Distributable, Serializable {
info.filePath = filePath;
info.blockOffset = blockOffset;
info.blockLength = blockLength;
- info.segmentId = segmentId;
+ info.segment = segment;
info.blockletId = blockletId;
info.locations = locations;
info.version = version;
@@ -229,7 +230,15 @@ public class TableBlockInfo implements Distributable, Serializable {
* @return the segmentId
*/
public String getSegmentId() {
- return segmentId;
+ if (segment == null) {
+ return null;
+ } else {
+ return segment.getSegmentNo();
+ }
+ }
+
+ public Segment getSegment() {
+ return segment;
}
/**
@@ -264,7 +273,7 @@ public class TableBlockInfo implements Distributable, Serializable {
return false;
}
TableBlockInfo other = (TableBlockInfo) obj;
- if (!segmentId.equals(other.segmentId)) {
+ if (!segment.equals(other.segment)) {
return false;
}
if (blockOffset != other.blockOffset) {
@@ -300,8 +309,8 @@ public class TableBlockInfo implements Distributable, Serializable {
// get the segment id
// converr seg ID to double.
- double seg1 = Double.parseDouble(segmentId);
- double seg2 = Double.parseDouble(((TableBlockInfo) other).segmentId);
+ double seg1 = Double.parseDouble(segment.getSegmentNo());
+ double seg2 = Double.parseDouble(((TableBlockInfo) other).segment.getSegmentNo());
if (seg1 - seg2 < 0) {
return -1;
}
@@ -358,7 +367,7 @@ public class TableBlockInfo implements Distributable, Serializable {
int result = filePath.hashCode();
result = 31 * result + (int) (blockOffset ^ (blockOffset >>> 32));
result = 31 * result + (int) (blockLength ^ (blockLength >>> 32));
- result = 31 * result + segmentId.hashCode();
+ result = 31 * result + segment.hashCode();
result = 31 * result + blockletInfos.getStartBlockletNumber();
return result;
}
@@ -457,7 +466,7 @@ public class TableBlockInfo implements Distributable, Serializable {
sb.append("filePath='").append(filePath).append('\'');
sb.append(", blockOffset=").append(blockOffset);
sb.append(", blockLength=").append(blockLength);
- sb.append(", segmentId='").append(segmentId).append('\'');
+ sb.append(", segment='").append(segment.toString()).append('\'');
sb.append(", blockletId='").append(blockletId).append('\'');
sb.append(", locations=").append(Arrays.toString(locations));
sb.append('}');
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index c434e2e..65fcb4b 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -122,7 +122,9 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
if (tableBlockIndexUniqueIdentifiers == null) {
tableBlockIndexUniqueIdentifiers =
BlockletDataMapUtil.getTableBlockUniqueIdentifiers(segment);
- segmentMap.put(segment.getSegmentNo(), tableBlockIndexUniqueIdentifiers);
+ if (tableBlockIndexUniqueIdentifiers.size() > 0) {
+ segmentMap.put(segment.getSegmentNo(), tableBlockIndexUniqueIdentifiers);
+ }
}
return tableBlockIndexUniqueIdentifiers;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
index acfc145..0b1c1e3 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
@@ -140,13 +140,16 @@ public class SegmentFileStore {
/**
* Write segment file to the metadata folder of the table
- * @param tablePath table path
+ *
+ * @param carbonTable CarbonTable
* @param segmentId segment id
- * @param UUID a UUID string used to construct the segment file name
+ * @param UUID a UUID string used to construct the segment file name
* @return segment file name
*/
- public static String writeSegmentFile(String tablePath, String segmentId, String UUID)
+ public static String writeSegmentFile(CarbonTable carbonTable, String segmentId, String UUID)
throws IOException {
+ String tablePath = carbonTable.getTablePath();
+ boolean supportFlatFolder = carbonTable.isSupportFlatFolder();
String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
CarbonFile segmentFolder = FileFactory.getCarbonFile(segmentPath);
CarbonFile[] indexFiles = segmentFolder.listFiles(new CarbonFileFilter() {
@@ -167,9 +170,12 @@ public class SegmentFileStore {
folderDetails.getFiles().add(file.getName());
}
}
- String segmentRelativePath = segmentPath.substring(tablePath.length(), segmentPath.length());
+ String segmentRelativePath = "/";
+ if (!supportFlatFolder) {
+ segmentRelativePath = segmentPath.substring(tablePath.length(), segmentPath.length());
+ }
segmentFile.addPath(segmentRelativePath, folderDetails);
- String segmentFileFolder = CarbonTablePath.getSegmentFilesLocation(tablePath);
+ String segmentFileFolder = CarbonTablePath.getSegmentFilesLocation(tablePath);
CarbonFile carbonFile = FileFactory.getCarbonFile(segmentFileFolder);
if (!carbonFile.exists()) {
carbonFile.mkdirs(segmentFileFolder, FileFactory.getFileType(segmentFileFolder));
@@ -177,12 +183,31 @@ public class SegmentFileStore {
String segmentFileName = genSegmentFileName(segmentId, UUID) + CarbonTablePath.SEGMENT_EXT;
// write segment info to new file.
writeSegmentFile(segmentFile, segmentFileFolder + File.separator + segmentFileName);
+
+ // Move all files to table path from segment folder.
+ if (supportFlatFolder) {
+ moveFromTempFolder(segmentPath, tablePath);
+ }
+
return segmentFileName;
}
return null;
}
/**
+ * Move the loaded data from source folder to destination folder.
+ */
+ private static void moveFromTempFolder(String source, String dest) {
+
+ CarbonFile oldFolder = FileFactory.getCarbonFile(source);
+ CarbonFile[] oldFiles = oldFolder.listFiles();
+ for (CarbonFile file : oldFiles) {
+ file.renameForce(dest + CarbonCommonConstants.FILE_SEPARATOR + file.getName());
+ }
+ oldFolder.delete();
+ }
+
+ /**
* Writes the segment file in json format
* @param segmentFile
* @param path
@@ -218,26 +243,37 @@ public class SegmentFileStore {
throws IOException {
CarbonFile[] segmentFiles = getSegmentFiles(readPath);
if (segmentFiles != null && segmentFiles.length > 0) {
- SegmentFile segmentFile = null;
- for (CarbonFile file : segmentFiles) {
- SegmentFile localSegmentFile = readSegmentFile(file.getAbsolutePath());
- if (segmentFile == null && localSegmentFile != null) {
- segmentFile = localSegmentFile;
- }
- if (localSegmentFile != null) {
- segmentFile = segmentFile.merge(localSegmentFile);
- }
- }
- if (segmentFile != null) {
- String path = writePath + "/" + mergeFileName + CarbonTablePath.SEGMENT_EXT;
- writeSegmentFile(segmentFile, path);
- FileFactory.deleteAllCarbonFilesOfDir(FileFactory.getCarbonFile(readPath));
- }
+ SegmentFile segmentFile = mergeSegmentFiles(mergeFileName, writePath, segmentFiles);
+ FileFactory.deleteAllCarbonFilesOfDir(FileFactory.getCarbonFile(readPath));
return segmentFile;
}
return null;
}
+ public static SegmentFile mergeSegmentFiles(String mergeFileName, String writePath,
+ CarbonFile[] segmentFiles) throws IOException {
+ SegmentFile segmentFile = null;
+ for (CarbonFile file : segmentFiles) {
+ SegmentFile localSegmentFile = readSegmentFile(file.getAbsolutePath());
+ if (segmentFile == null && localSegmentFile != null) {
+ segmentFile = localSegmentFile;
+ }
+ if (localSegmentFile != null) {
+ segmentFile = segmentFile.merge(localSegmentFile);
+ }
+ }
+ if (segmentFile != null) {
+ String path = writePath + "/" + mergeFileName + CarbonTablePath.SEGMENT_EXT;
+ writeSegmentFile(segmentFile, path);
+ }
+ return segmentFile;
+ }
+
+ public static String getSegmentFilePath(String tablePath, String segmentFileName) {
+ return CarbonTablePath.getSegmentFilesLocation(tablePath) +
+ CarbonCommonConstants.FILE_SEPARATOR + segmentFileName;
+ }
+
/**
* This API will update the segmentFile of a passed segment.
*
@@ -248,6 +284,9 @@ public class SegmentFileStore {
throws IOException {
boolean status = false;
String tableStatusPath = CarbonTablePath.getTableStatusFilePath(tablePath);
+ if (!FileFactory.isFileExist(tableStatusPath)) {
+ return status;
+ }
String metadataPath = CarbonTablePath.getMetadataPath(tablePath);
AbsoluteTableIdentifier absoluteTableIdentifier =
AbsoluteTableIdentifier.from(tablePath, null, null);
@@ -654,7 +693,10 @@ public class SegmentFileStore {
new SegmentFileStore(table.getTablePath(), segment.getSegmentFile());
fileStore.readIndexFiles(SegmentStatus.MARKED_FOR_DELETE, false);
if (forceDelete) {
- deletePhysicalPartition(partitionSpecs, fileStore.getIndexFilesMap());
+ deletePhysicalPartition(
+ partitionSpecs,
+ fileStore.getIndexFilesMap(),
+ table.getTablePath());
}
for (Map.Entry<String, List<String>> entry : fileStore.indexFilesMap.entrySet()) {
String indexFile = entry.getKey();
@@ -698,7 +740,7 @@ public class SegmentFileStore {
FileFactory.deleteFile(file, FileFactory.getFileType(file));
}
}
- deletePhysicalPartition(partitionSpecs, indexFilesMap);
+ deletePhysicalPartition(partitionSpecs, indexFilesMap, tablePath);
String segmentFilePath =
CarbonTablePath.getSegmentFilesLocation(tablePath) + CarbonCommonConstants.FILE_SEPARATOR
+ segmentFile;
@@ -713,7 +755,7 @@ public class SegmentFileStore {
* If partition specs are null, then directly delete parent directory in locationMap.
*/
private static void deletePhysicalPartition(List<PartitionSpec> partitionSpecs,
- Map<String, List<String>> locationMap) throws IOException {
+ Map<String, List<String>> locationMap, String tablePath) throws IOException {
for (Map.Entry<String, List<String>> entry : locationMap.entrySet()) {
if (partitionSpecs != null) {
Path location = new Path(entry.getKey());
@@ -733,7 +775,8 @@ public class SegmentFileStore {
Path location = new Path(entry.getKey()).getParent();
// delete the segment folder
CarbonFile segmentPath = FileFactory.getCarbonFile(location.toString());
- if (null != segmentPath) {
+ if (null != segmentPath && segmentPath.exists() &&
+ !new Path(tablePath).equals(new Path(segmentPath.getAbsolutePath()))) {
FileFactory.deleteAllCarbonFilesOfDir(segmentPath);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 20bc7a1..f48ada0 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -1008,6 +1008,19 @@ public class CarbonTable implements Serializable {
}
/**
+ * Whether this table supports flat folder structure, it means all data files directly written
+ * under table path
+ */
+ public boolean isSupportFlatFolder() {
+ boolean supportFlatFolder = Boolean.parseBoolean(CarbonCommonConstants.DEFAULT_FLAT_FOLDER);
+ Map<String, String> tblProps = getTableInfo().getFactTable().getTableProperties();
+ if (tblProps.containsKey(CarbonCommonConstants.FLAT_FOLDER)) {
+ supportFlatFolder = tblProps.get(CarbonCommonConstants.FLAT_FOLDER).equalsIgnoreCase("true");
+ }
+ return supportFlatFolder;
+ }
+
+ /**
* update the carbon table by using the passed tableInfo
*
* @param table
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index 1d0ef44..40d498c 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -47,6 +47,8 @@ import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.hadoop.fs.Path;
+
/**
* This class contains all update utility methods
*/
@@ -78,20 +80,17 @@ public class CarbonUpdateUtil {
/**
* Returns block path from tuple id
- *
- * @param tid
- * @param factPath
- * @return
*/
- public static String getTableBlockPath(String tid, String factPath, boolean isPartitionTable) {
+ public static String getTableBlockPath(String tid, String tablePath, boolean isSegmentFile) {
String partField = getRequiredFieldFromTID(tid, TupleIdEnum.PART_ID);
- if (isPartitionTable) {
- return factPath + CarbonCommonConstants.FILE_SEPARATOR + partField;
+ // If it has segment file then partfield can be appended directly to table path
+ if (isSegmentFile) {
+ return tablePath + CarbonCommonConstants.FILE_SEPARATOR + partField.replace("#", "/");
}
String part = CarbonTablePath.addPartPrefix(partField);
String segment =
CarbonTablePath.addSegmentPrefix(getRequiredFieldFromTID(tid, TupleIdEnum.SEGMENT_ID));
- return factPath + CarbonCommonConstants.FILE_SEPARATOR + part
+ return CarbonTablePath.getFactDir(tablePath) + CarbonCommonConstants.FILE_SEPARATOR + part
+ CarbonCommonConstants.FILE_SEPARATOR + segment;
}
@@ -386,29 +385,45 @@ public class CarbonUpdateUtil {
return segmentName.split(CarbonCommonConstants.UNDERSCORE)[1];
}
- public static long getLatestTaskIdForSegment(String segmentId, String tablePath) {
- String segmentDirPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
+ public static long getLatestTaskIdForSegment(Segment segment, String tablePath)
+ throws IOException {
+ long max = 0;
+ List<String> dataFiles = new ArrayList<>();
+ if (segment.getSegmentFileName() != null) {
+ SegmentFileStore fileStore = new SegmentFileStore(tablePath, segment.getSegmentFileName());
+ fileStore.readIndexFiles();
+ Map<String, List<String>> indexFilesMap = fileStore.getIndexFilesMap();
+ List<String> dataFilePaths = new ArrayList<>();
+ for (List<String> paths : indexFilesMap.values()) {
+ dataFilePaths.addAll(paths);
+ }
+ for (String dataFilePath : dataFilePaths) {
+ dataFiles.add(new Path(dataFilePath).getName());
+ }
- // scan all the carbondata files and get the latest task ID.
- CarbonFile segment =
- FileFactory.getCarbonFile(segmentDirPath, FileFactory.getFileType(segmentDirPath));
- CarbonFile[] dataFiles = segment.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
+ } else {
+ String segmentDirPath = CarbonTablePath.getSegmentPath(tablePath, segment.getSegmentNo());
+ // scan all the carbondata files and get the latest task ID.
+ CarbonFile segmentDir =
+ FileFactory.getCarbonFile(segmentDirPath, FileFactory.getFileType(segmentDirPath));
+ CarbonFile[] carbonDataFiles = segmentDir.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
- if (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
- return true;
+ if (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
+ return true;
+ }
+ return false;
}
- return false;
+ });
+ for (CarbonFile carbonDataFile : carbonDataFiles) {
+ dataFiles.add(carbonDataFile.getName());
}
- });
- long max = 0;
- if (null != dataFiles) {
- for (CarbonFile file : dataFiles) {
- long taskNumber =
- Long.parseLong(CarbonTablePath.DataFileUtil.getTaskNo(file.getName()).split("_")[0]);
- if (taskNumber > max) {
- max = taskNumber;
- }
+ }
+ for (String name : dataFiles) {
+ long taskNumber =
+ Long.parseLong(CarbonTablePath.DataFileUtil.getTaskNo(name).split("_")[0]);
+ if (taskNumber > max) {
+ max = taskNumber;
}
}
// return max task No
@@ -562,7 +577,7 @@ public class CarbonUpdateUtil {
List<Segment> segmentFilesToBeUpdatedLatest = new ArrayList<>();
for (Segment segment : segmentFilesToBeUpdated) {
String file =
- SegmentFileStore.writeSegmentFile(table.getTablePath(), segment.getSegmentNo(), UUID);
+ SegmentFileStore.writeSegmentFile(table, segment.getSegmentNo(), UUID);
segmentFilesToBeUpdatedLatest.add(new Segment(segment.getSegmentNo(), file));
}
if (segmentFilesToBeUpdated.size() > 0) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index ff0e5ce..2bbe75c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -34,6 +34,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.common.logging.impl.StandardLogService;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
@@ -68,6 +69,7 @@ import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.commons.lang3.ArrayUtils;
@@ -256,7 +258,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
dataRefNode.numberOfNodes(),
dataRefNode.getBlockInfos().get(0).getFilePath(),
dataRefNode.getBlockInfos().get(0).getDeletedDeltaFilePath(),
- dataRefNode.getBlockInfos().get(0).getSegmentId()));
+ dataRefNode.getBlockInfos().get(0).getSegment()));
}
if (null != queryModel.getStatisticsRecorder()) {
QueryStatistic queryStatistic = new QueryStatistic();
@@ -278,7 +280,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
*/
private BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel,
AbstractIndex blockIndex, int startBlockletIndex, int numberOfBlockletToScan, String filePath,
- String[] deleteDeltaFiles, String segmentId)
+ String[] deleteDeltaFiles, Segment segment)
throws QueryExecutionException {
BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
SegmentProperties segmentProperties = blockIndex.getSegmentProperties();
@@ -291,9 +293,15 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
queryModel.getProjectionDimensions(), tableBlockDimensions,
segmentProperties.getComplexDimensions(), queryModel.getProjectionMeasures().size(),
queryModel.getTable().getTableInfo().isTransactionalTable());
- blockExecutionInfo.setBlockId(
- CarbonUtil.getBlockId(queryModel.getAbsoluteTableIdentifier(), filePath, segmentId,
- queryModel.getTable().getTableInfo().isTransactionalTable()));
+ String blockId = CarbonUtil
+ .getBlockId(queryModel.getAbsoluteTableIdentifier(), filePath, segment.getSegmentNo(),
+ queryModel.getTable().getTableInfo().isTransactionalTable(),
+ segment.getSegmentFileName() != null);
+ if (segment.getSegmentFileName() != null) {
+ blockExecutionInfo.setBlockId(CarbonTablePath.getShortBlockIdForPartitionTable(blockId));
+ } else {
+ blockExecutionInfo.setBlockId(CarbonTablePath.getShortBlockId(blockId));
+ }
blockExecutionInfo.setDeleteDeltaFilePath(deleteDeltaFiles);
blockExecutionInfo.setStartBlockletIndex(startBlockletIndex);
blockExecutionInfo.setNumberOfBlockletToScan(numberOfBlockletToScan);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
index b85945f..eadd502 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
@@ -45,7 +45,6 @@ import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
import org.apache.carbondata.core.stats.QueryStatisticsModel;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
/**
* Scanned result class which will store and provide the result on request
@@ -525,7 +524,7 @@ public abstract class BlockletScannedResult {
* "Part0/Segment_0/part-0-0_batchno0-0-1517155583332.carbondata/0"
*/
public void setBlockletId(String blockletId) {
- this.blockletId = CarbonTablePath.getShortBlockId(blockletId);
+ this.blockletId = blockletId;
blockletNumber = CarbonUpdateUtil.getRequiredFieldFromTID(blockletId, TupleIdEnum.BLOCKLET_ID);
// if deleted recors map is present for this block
// then get the first page deleted vo
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
index a48804c..1665ce6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
@@ -82,9 +82,9 @@ public class BlockletFullScanner implements BlockletScanner {
.get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED,
totalPagesScanned.getCount() + rawBlockletColumnChunks.getDataBlock().numberOfPages());
- scannedResult.setBlockletId(
- blockExecutionInfo.getBlockIdString() + CarbonCommonConstants.FILE_SEPARATOR +
- rawBlockletColumnChunks.getDataBlock().blockletIndex());
+ String blockletId = blockExecutionInfo.getBlockIdString() + CarbonCommonConstants.FILE_SEPARATOR
+ + rawBlockletColumnChunks.getDataBlock().blockletIndex();
+ scannedResult.setBlockletId(blockletId);
if (!blockExecutionInfo.isPrefetchBlocklet()) {
readBlocklet(rawBlockletColumnChunks);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index c2faadc..eb850e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -69,7 +69,6 @@ public class SegmentUpdateStatusManager {
private LoadMetadataDetails[] segmentDetails;
private SegmentUpdateDetails[] updateDetails;
private Map<String, SegmentUpdateDetails> blockAndDetailsMap;
- private boolean isPartitionTable;
public SegmentUpdateStatusManager(CarbonTable table,
LoadMetadataDetails[] segmentDetails) {
@@ -77,7 +76,6 @@ public class SegmentUpdateStatusManager {
// current it is used only for read function scenarios, as file update always requires to work
// on latest file status.
this.segmentDetails = segmentDetails;
- isPartitionTable = table.isHivePartitionTable();
updateDetails = readLoadMetadata();
populateMap();
}
@@ -93,7 +91,6 @@ public class SegmentUpdateStatusManager {
segmentDetails = SegmentStatusManager.readLoadMetadata(
CarbonTablePath.getMetadataPath(identifier.getTablePath()));
}
- isPartitionTable = table.isHivePartitionTable();
if (segmentDetails.length != 0) {
updateDetails = readLoadMetadata();
} else {
@@ -249,27 +246,37 @@ public class SegmentUpdateStatusManager {
* @throws Exception
*/
public String[] getDeleteDeltaFilePath(String blockFilePath, String segmentId) throws Exception {
- String blockId = CarbonUtil.getBlockId(identifier, blockFilePath, segmentId, true);
+ String segmentFile = null;
+ for (LoadMetadataDetails segmentDetail : segmentDetails) {
+ if (segmentDetail.getLoadName().equals(segmentId)) {
+ segmentFile = segmentDetail.getSegmentFile();
+ break;
+ }
+ }
+ String blockId =
+ CarbonUtil.getBlockId(identifier, blockFilePath, segmentId, true, segmentFile != null);
String tupleId;
- if (isPartitionTable) {
+ if (segmentFile != null) {
tupleId = CarbonTablePath.getShortBlockIdForPartitionTable(blockId);
} else {
tupleId = CarbonTablePath.getShortBlockId(blockId);
}
- return getDeltaFiles(tupleId, CarbonCommonConstants.DELETE_DELTA_FILE_EXT)
+ return getDeltaFiles(tupleId, CarbonCommonConstants.DELETE_DELTA_FILE_EXT, segmentFile)
.toArray(new String[0]);
}
/**
* Returns all delta file paths of specified block
*/
- private List<String> getDeltaFiles(String tupleId, String extension) throws Exception {
+ private List<String> getDeltaFiles(String tupleId, String extension, String segmentFile)
+ throws Exception {
String segment = CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.SEGMENT_ID);
String completeBlockName = CarbonTablePath.addDataPartPrefix(
CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.BLOCK_ID)
+ CarbonCommonConstants.FACT_FILE_EXT);
+
String blockPath;
- if (isPartitionTable) {
+ if (segmentFile != null) {
blockPath = identifier.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR
+ CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.PART_ID)
.replace("#", "/") + CarbonCommonConstants.FILE_SEPARATOR + completeBlockName;
@@ -283,7 +290,8 @@ public class SegmentUpdateStatusManager {
if (!file.exists()) {
throw new Exception("Invalid tuple id " + tupleId);
}
- String blockNameWithoutExtn = completeBlockName.substring(0, completeBlockName.indexOf('.'));
+ String blockNameWithoutExtn =
+ completeBlockName.substring(0, completeBlockName.lastIndexOf('.'));
//blockName without timestamp
final String blockNameFromTuple =
blockNameWithoutExtn.substring(0, blockNameWithoutExtn.lastIndexOf("-"));
@@ -363,15 +371,15 @@ public class SegmentUpdateStatusManager {
public boolean accept(CarbonFile pathName) {
String fileName = pathName.getName();
if (fileName.endsWith(extension) && pathName.getSize() > 0) {
- String firstPart = fileName.substring(0, fileName.indexOf('.'));
+ String firstPart = fileName.substring(0, fileName.lastIndexOf('.'));
String blockName =
- firstPart.substring(0, firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN));
+ firstPart.substring(0, firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN));
long timestamp = Long.parseLong(firstPart
- .substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
- firstPart.length()));
+ .substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
+ firstPart.length()));
if (blockNameFromTuple.equals(blockName) && (
- (Long.compare(timestamp, deltaEndTimeStamp) <= 0) && (
- Long.compare(timestamp, deltaStartTimestamp) >= 0))) {
+ (Long.compare(timestamp, deltaEndTimeStamp) <= 0) && (
+ Long.compare(timestamp, deltaStartTimestamp) >= 0))) {
return true;
}
}
@@ -479,7 +487,7 @@ public class SegmentUpdateStatusManager {
String fileName = eachFile.getName();
if (fileName.endsWith(fileExtension)) {
- String firstPart = fileName.substring(0, fileName.indexOf('.'));
+ String firstPart = fileName.substring(0, fileName.lastIndexOf('.'));
long timestamp = Long.parseLong(firstPart
.substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 2aa4a05..836b193 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2973,24 +2973,28 @@ public final class CarbonUtil {
* @return
*/
public static String getBlockId(AbsoluteTableIdentifier identifier, String filePath,
- String segmentId, boolean isTransactionalTable) {
+ String segmentId, boolean isTransactionalTable, boolean hasSegmentFile) {
String blockId;
String blockName = filePath.substring(filePath.lastIndexOf("/") + 1, filePath.length());
String tablePath = identifier.getTablePath();
if (filePath.startsWith(tablePath)) {
- String factDir = CarbonTablePath.getFactDir(tablePath);
- if (filePath.startsWith(factDir) || !isTransactionalTable) {
+ if (!isTransactionalTable || !hasSegmentFile) {
blockId = "Part0" + CarbonCommonConstants.FILE_SEPARATOR + "Segment_" + segmentId
+ CarbonCommonConstants.FILE_SEPARATOR + blockName;
} else {
// This is the case with partition table.
- String partitionDir =
- filePath.substring(tablePath.length() + 1, filePath.length() - blockName.length() - 1);
-
+ String partitionDir;
+ if (tablePath.length() + 1 < filePath.length() - blockName.length() - 1) {
+ partitionDir =
+ filePath.substring(tablePath.length() + 1,
+ filePath.length() - blockName.length() - 1);
+ } else {
+ partitionDir = "";
+ }
// Replace / with # on partition director to support multi level partitioning. And access
// them all as a single entity.
- blockId = partitionDir.replace("/", "#") + CarbonCommonConstants.FILE_SEPARATOR + "Segment_"
+ blockId = partitionDir.replace("/", "#") + CarbonCommonConstants.FILE_SEPARATOR
+ segmentId + CarbonCommonConstants.FILE_SEPARATOR + blockName;
}
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index e8a121c..fe68adf 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -228,7 +228,7 @@ public class CarbonTablePath {
public static String getCarbonDataFilePath(String tablePath, String segmentId, Integer filePartNo,
Long taskNo, int batchNo, int bucketNumber, String factUpdateTimeStamp) {
return getSegmentPath(tablePath, segmentId) + File.separator + getCarbonDataFileName(
- filePartNo, taskNo, bucketNumber, batchNo, factUpdateTimeStamp);
+ filePartNo, taskNo, bucketNumber, batchNo, factUpdateTimeStamp, segmentId);
}
/**
@@ -283,7 +283,7 @@ public class CarbonTablePath {
default:
String segmentDir = getSegmentPath(tablePath, segmentId);
return segmentDir + File.separator + getCarbonIndexFileName(taskId,
- Integer.parseInt(bucketNumber), timeStamp);
+ Integer.parseInt(bucketNumber), timeStamp, segmentId);
}
}
@@ -297,16 +297,17 @@ public class CarbonTablePath {
default:
String segmentDir = getSegmentPath(tablePath, segmentId);
return segmentDir + File.separator + getCarbonIndexFileName(Long.parseLong(taskId),
- Integer.parseInt(bucketNumber), batchNo, timeStamp);
+ Integer.parseInt(bucketNumber), batchNo, timeStamp, segmentId);
}
}
private static String getCarbonIndexFileName(String taskNo, int bucketNumber,
- String factUpdatedtimeStamp) {
+ String factUpdatedtimeStamp, String segmentNo) {
if (bucketNumber == -1) {
- return taskNo + "-" + factUpdatedtimeStamp + INDEX_FILE_EXT;
+ return taskNo + "-" + segmentNo + "-" + factUpdatedtimeStamp + INDEX_FILE_EXT;
}
- return taskNo + "-" + bucketNumber + "-" + factUpdatedtimeStamp + INDEX_FILE_EXT;
+ return taskNo + "-" + bucketNumber + "-" + segmentNo + "-" + factUpdatedtimeStamp
+ + INDEX_FILE_EXT;
}
/**
@@ -325,14 +326,15 @@ public class CarbonTablePath {
* @return gets data file name only with out path
*/
public static String getCarbonDataFileName(Integer filePartNo, Long taskNo, int bucketNumber,
- int batchNo, String factUpdateTimeStamp) {
+ int batchNo, String factUpdateTimeStamp, String segmentNo) {
return DATA_PART_PREFIX + filePartNo + "-" + taskNo + BATCH_PREFIX + batchNo + "-"
- + bucketNumber + "-" + factUpdateTimeStamp + CARBON_DATA_EXT;
+ + bucketNumber + "-" + segmentNo + "-" + factUpdateTimeStamp + CARBON_DATA_EXT;
}
public static String getShardName(Long taskNo, int bucketNumber, int batchNo,
- String factUpdateTimeStamp) {
- return taskNo + BATCH_PREFIX + batchNo + "-" + bucketNumber + "-" + factUpdateTimeStamp;
+ String factUpdateTimeStamp, String segmentNo) {
+ return taskNo + BATCH_PREFIX + batchNo + "-" + bucketNumber + "-" + segmentNo + "-"
+ + factUpdateTimeStamp;
}
/**
@@ -343,13 +345,13 @@ public class CarbonTablePath {
* @return filename
*/
public static String getCarbonIndexFileName(long taskNo, int bucketNumber, int batchNo,
- String factUpdatedTimeStamp) {
- return taskNo + BATCH_PREFIX + batchNo + "-" + bucketNumber + "-" + factUpdatedTimeStamp
+ String factUpdatedTimeStamp, String segmentNo) {
+ return getShardName(taskNo, bucketNumber, batchNo, factUpdatedTimeStamp, segmentNo)
+ INDEX_FILE_EXT;
}
public static String getCarbonStreamIndexFileName() {
- return getCarbonIndexFileName(0, 0, 0, "0");
+ return getCarbonIndexFileName(0, 0, 0, "0", "0");
}
public static String getCarbonStreamIndexFilePath(String segmentDir) {
@@ -408,11 +410,25 @@ public class CarbonTablePath {
public static String getDataMapStorePathOnShardName(String tablePath, String segmentId,
String dataMapName, String shardName) {
return new StringBuilder()
- .append(getSegmentPath(tablePath, segmentId))
+ .append(getDataMapStorePath(tablePath, segmentId, dataMapName))
+ .append(File.separator)
+ .append(shardName)
+ .toString();
+ }
+
+ /**
+ * Return store path for datamap based on the dataMapName,
+ *
+ * @return store path based on datamapname
+ */
+ public static String getDataMapStorePath(String tablePath, String segmentId,
+ String dataMapName) {
+ return new StringBuilder()
+ .append(tablePath)
.append(File.separator)
.append(dataMapName)
.append(File.separator)
- .append(shardName)
+ .append(segmentId)
.toString();
}
@@ -517,6 +533,29 @@ public class CarbonTablePath {
}
/**
+ * Return the updated timestamp information from given carbon data file name
+ */
+ public static String getSegmentNo(String carbonDataFileName) {
+ // Get the file name from path
+ String fileName = getFileName(carbonDataFileName);
+ // + 1 for size of "-"
+ int firstDashPos = fileName.indexOf("-");
+ int startIndex1 = fileName.indexOf("-", firstDashPos + 1) + 1;
+ int endIndex1 = fileName.indexOf("-", startIndex1);
+ int startIndex = fileName.indexOf("-", endIndex1 + 1);
+ if (startIndex > -1) {
+ startIndex += 1;
+ int endIndex = fileName.indexOf("-", startIndex);
+ if (endIndex == -1) {
+ return null;
+ }
+ return fileName.substring(startIndex, endIndex);
+ } else {
+ return null;
+ }
+ }
+
+ /**
* Return the taskId part from taskNo(include taskId + batchNo)
*/
public static long getTaskIdFromTaskNo(String taskNo) {
@@ -545,7 +584,7 @@ public class CarbonTablePath {
/**
* gets segement id from given absolute data file path
*/
- public static String getSegmentId(String dataFileAbsolutePath) {
+ public static String getSegmentIdFromPath(String dataFileAbsolutePath) {
// find segment id from last of data file path
String tempdataFileAbsolutePath = dataFileAbsolutePath.replace(
CarbonCommonConstants.WINDOWS_FILE_SEPARATOR, CarbonCommonConstants.FILE_SEPARATOR);
@@ -647,8 +686,7 @@ public class CarbonTablePath {
* @return shortBlockId
*/
public static String getShortBlockIdForPartitionTable(String blockId) {
- return blockId.replace(SEGMENT_PREFIX, "")
- .replace(DATA_PART_PREFIX, "")
+ return blockId.replace(DATA_PART_PREFIX, "")
.replace(CARBON_DATA_EXT, "");
}
@@ -687,7 +725,8 @@ public class CarbonTablePath {
*/
public static String getShardName(String actualBlockName) {
return DataFileUtil.getTaskNo(actualBlockName) + "-" + DataFileUtil.getBucketNo(actualBlockName)
- + "-" + DataFileUtil.getTimeStampFromFileName(actualBlockName);
+ + "-" + DataFileUtil.getSegmentNo(actualBlockName) + "-" + DataFileUtil
+ .getTimeStampFromFileName(actualBlockName);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/test/java/org/apache/carbondata/core/datastore/block/TableBlockInfoTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/TableBlockInfoTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/TableBlockInfoTest.java
index ecdaf3d..e61ea6f 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/TableBlockInfoTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/TableBlockInfoTest.java
@@ -169,7 +169,7 @@ public class TableBlockInfoTest {
@Test public void hashCodeTest() {
int res = tableBlockInfo.hashCode();
- int expectedResult = 1041505621;
+ int expectedResult = 1041506582;
assertEquals(res, expectedResult);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
index 4293536..e52c737 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
@@ -34,26 +34,6 @@ public class CarbonFormatDirectoryStructureTest {
private final String CARBON_STORE = "/opt/carbonstore";
- /**
- * test table path methods
- */
- @Test public void testTablePathStructure() throws IOException {
- CarbonTableIdentifier tableIdentifier = new CarbonTableIdentifier("d1", "t1", UUID.randomUUID().toString());
- AbsoluteTableIdentifier identifier =
- AbsoluteTableIdentifier.from(CARBON_STORE + "/d1/t1", tableIdentifier);
- assertTrue(identifier.getTablePath().replace("\\", "/").equals(CARBON_STORE + "/d1/t1"));
- assertTrue(CarbonTablePath.getSchemaFilePath(identifier.getTablePath()).replace("\\", "/").equals(CARBON_STORE + "/d1/t1/Metadata/schema"));
- assertTrue(CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()).replace("\\", "/")
- .equals(CARBON_STORE + "/d1/t1/Metadata/tablestatus"));
- assertTrue(CarbonTablePath.getDictionaryFilePath(identifier.getTablePath(), "t1_c1").replace("\\", "/")
- .equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.dict"));
- assertTrue(CarbonTablePath.getDictionaryMetaFilePath(identifier.getTablePath(), "t1_c1").replace("\\", "/")
- .equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.dictmeta"));
- assertTrue(CarbonTablePath.getSortIndexFilePath(identifier.getTablePath(),"t1_c1").replace("\\", "/")
- .equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.sortindex"));
- assertTrue(CarbonTablePath.getCarbonDataFilePath(identifier.getTablePath(), "2", 3, 4L, 0, 0, "999").replace("\\", "/")
- .equals(CARBON_STORE + "/d1/t1/Fact/Part0/Segment_2/part-3-4_batchno0-0-999.carbondata"));
- }
/**
* test data file name
@@ -67,5 +47,7 @@ public class CarbonFormatDirectoryStructureTest {
assertTrue(CarbonTablePath.DataFileUtil.getTaskNo("/opt/apache-carbon/part-3-4-999.carbondata").equals("4"));
assertTrue(
CarbonTablePath.DataFileUtil.getTimeStampFromFileName("/opt/apache-carbon/part-3-4-999.carbondata").equals("999"));
+ assertTrue(CarbonTablePath.DataFileUtil.getSegmentNo("part-3-4-0-999.carbondata") == null);
+ assertTrue(CarbonTablePath.DataFileUtil.getSegmentNo("part-3-4-0-0-999.carbondata").equals("0"));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
index 3231551..cda49b3 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
@@ -16,7 +16,6 @@
*/
package org.apache.carbondata.datamap.bloom;
-import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -259,8 +258,8 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa
if (dataMaps.size() > 0) {
for (TableDataMap dataMap : dataMaps) {
List<CarbonFile> indexFiles;
- String dmPath = CarbonTablePath.getSegmentPath(tablePath, segmentId) + File.separator
- + dataMap.getDataMapSchema().getDataMapName();
+ String dmPath = CarbonTablePath
+ .getDataMapStorePath(tablePath, segmentId, dataMap.getDataMapSchema().getDataMapName());
FileFactory.FileType fileType = FileFactory.getFileType(dmPath);
final CarbonFile dirPath = FileFactory.getCarbonFile(dmPath, fileType);
indexFiles = Arrays.asList(dirPath.listFiles(new CarbonFileFilter() {
@@ -323,9 +322,8 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa
List<Segment> validSegments = ssm.getValidAndInvalidSegments().getValidSegments();
for (Segment segment : validSegments) {
String segmentId = segment.getSegmentNo();
- String datamapPath = CarbonTablePath.getSegmentPath(
- getCarbonTable().getAbsoluteTableIdentifier().getTablePath(), segmentId)
- + File.separator + dataMapName;
+ String datamapPath = CarbonTablePath
+ .getDataMapStorePath(getCarbonTable().getTablePath(), segmentId, dataMapName);
if (FileFactory.isFileExist(datamapPath)) {
CarbonFile file = FileFactory.getCarbonFile(datamapPath,
FileFactory.getFileType(datamapPath));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
index 1da8edd..cc14dc4 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
@@ -17,7 +17,6 @@
package org.apache.carbondata.datamap.lucene;
-import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -182,9 +181,8 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> extends DataMapFactor
List<Segment> validSegments = ssm.getValidAndInvalidSegments().getValidSegments();
for (Segment segment : validSegments) {
String segmentId = segment.getSegmentNo();
- String datamapPath =
- CarbonTablePath.getSegmentPath(tableIdentifier.getTablePath(), segmentId)
- + File.separator + dataMapName;
+ String datamapPath = CarbonTablePath
+ .getDataMapStorePath(tableIdentifier.getTablePath(), segmentId, dataMapName);
if (FileFactory.isFileExist(datamapPath)) {
CarbonFile file =
FileFactory.getCarbonFile(datamapPath, FileFactory.getFileType(datamapPath));
@@ -227,9 +225,9 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> extends DataMapFactor
getAllIndexDirs(tableIdentifier.getTablePath(), segment.getSegmentNo());
if (segment.getFilteredIndexShardNames().size() == 0) {
for (CarbonFile indexDir : indexDirs) {
- DataMapDistributable luceneDataMapDistributable = new LuceneDataMapDistributable(
- CarbonTablePath.getSegmentPath(tableIdentifier.getTablePath(), segment.getSegmentNo()),
- indexDir.getAbsolutePath());
+ DataMapDistributable luceneDataMapDistributable =
+ new LuceneDataMapDistributable(tableIdentifier.getTablePath(),
+ indexDir.getAbsolutePath());
lstDataMapDistribute.add(luceneDataMapDistributable);
}
return lstDataMapDistribute;
@@ -303,9 +301,8 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> extends DataMapFactor
if (dataMaps.size() > 0) {
for (TableDataMap dataMap : dataMaps) {
List<CarbonFile> indexFiles;
- String dmPath =
- CarbonTablePath.getSegmentPath(tablePath, segmentId) + File.separator + dataMap
- .getDataMapSchema().getDataMapName();
+ String dmPath = CarbonTablePath
+ .getDataMapStorePath(tablePath, segmentId, dataMap.getDataMapSchema().getDataMapName());
FileFactory.FileType fileType = FileFactory.getFileType(dmPath);
final CarbonFile dirPath = FileFactory.getCarbonFile(dmPath, fileType);
indexFiles = Arrays.asList(dirPath.listFiles(new CarbonFileFilter() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index 02d272e..405ff53 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -54,7 +54,7 @@ public class CarbonInputSplit extends FileSplit
private static final long serialVersionUID = 3520344046772190207L;
public String taskId;
- private String segmentId;
+ private Segment segment;
private String bucketId;
@@ -91,7 +91,7 @@ public class CarbonInputSplit extends FileSplit
private String dataMapWritePath;
public CarbonInputSplit() {
- segmentId = null;
+ segment = null;
taskId = "0";
bucketId = "0";
blockletId = "0";
@@ -104,7 +104,7 @@ public class CarbonInputSplit extends FileSplit
String[] locations, ColumnarFormatVersion version, String[] deleteDeltaFiles,
String dataMapWritePath) {
super(path, start, length, locations);
- this.segmentId = segmentId;
+ this.segment = Segment.toSegment(segmentId);
String taskNo = CarbonTablePath.DataFileUtil.getTaskNo(path.getName());
if (taskNo.contains("_")) {
taskNo = taskNo.split("_")[0];
@@ -128,7 +128,7 @@ public class CarbonInputSplit extends FileSplit
public CarbonInputSplit(String segmentId, Path path, long start, long length, String[] locations,
FileFormat fileFormat) {
super(path, start, length, locations);
- this.segmentId = segmentId;
+ this.segment = Segment.toSegment(segmentId);
this.fileFormat = fileFormat;
taskId = "0";
bucketId = "0";
@@ -141,7 +141,7 @@ public class CarbonInputSplit extends FileSplit
public CarbonInputSplit(String segmentId, Path path, long start, long length, String[] locations,
String[] inMemoryHosts, FileFormat fileFormat) {
super(path, start, length, locations, inMemoryHosts);
- this.segmentId = segmentId;
+ this.segment = Segment.toSegment(segmentId);
this.fileFormat = fileFormat;
taskId = "0";
bucketId = "0";
@@ -184,8 +184,8 @@ public class CarbonInputSplit extends FileSplit
try {
TableBlockInfo blockInfo =
new TableBlockInfo(split.getPath().toString(), split.blockletId, split.getStart(),
- split.getSegmentId(), split.getLocations(), split.getLength(), blockletInfos,
- split.getVersion(), split.getDeleteDeltaFiles());
+ split.getSegment().toString(), split.getLocations(), split.getLength(),
+ blockletInfos, split.getVersion(), split.getDeleteDeltaFiles());
blockInfo.setDetailInfo(split.getDetailInfo());
blockInfo.setDataMapWriterPath(split.dataMapWritePath);
blockInfo.setBlockOffset(split.getDetailInfo().getBlockFooterOffset());
@@ -203,7 +203,7 @@ public class CarbonInputSplit extends FileSplit
try {
TableBlockInfo blockInfo =
new TableBlockInfo(inputSplit.getPath().toString(), inputSplit.blockletId,
- inputSplit.getStart(), inputSplit.getSegmentId(), inputSplit.getLocations(),
+ inputSplit.getStart(), inputSplit.getSegment().toString(), inputSplit.getLocations(),
inputSplit.getLength(), blockletInfos, inputSplit.getVersion(),
inputSplit.getDeleteDeltaFiles());
blockInfo.setDetailInfo(inputSplit.getDetailInfo());
@@ -215,12 +215,21 @@ public class CarbonInputSplit extends FileSplit
}
public String getSegmentId() {
- return segmentId;
+ if (segment != null) {
+ return segment.getSegmentNo();
+ } else {
+ return null;
+ }
}
+ public Segment getSegment() {
+ return segment;
+ }
+
+
@Override public void readFields(DataInput in) throws IOException {
super.readFields(in);
- this.segmentId = in.readUTF();
+ this.segment = Segment.toSegment(in.readUTF());
this.version = ColumnarFormatVersion.valueOf(in.readShort());
this.bucketId = in.readUTF();
this.blockletId = in.readUTF();
@@ -247,7 +256,7 @@ public class CarbonInputSplit extends FileSplit
@Override public void write(DataOutput out) throws IOException {
super.write(out);
- out.writeUTF(segmentId);
+ out.writeUTF(segment.toString());
out.writeShort(version.number());
out.writeUTF(bucketId);
out.writeUTF(blockletId);
@@ -323,7 +332,7 @@ public class CarbonInputSplit extends FileSplit
// get the segment id
// converr seg ID to double.
- double seg1 = Double.parseDouble(segmentId);
+ double seg1 = Double.parseDouble(segment.getSegmentNo());
double seg2 = Double.parseDouble(other.getSegmentId());
if (seg1 - seg2 < 0) {
return -1;
@@ -381,7 +390,7 @@ public class CarbonInputSplit extends FileSplit
@Override public int hashCode() {
int result = taskId.hashCode();
- result = 31 * result + segmentId.hashCode();
+ result = 31 * result + segment.hashCode();
result = 31 * result + bucketId.hashCode();
result = 31 * result + invalidSegments.hashCode();
result = 31 * result + numberOfBlocklets;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
index 485b087..3688026 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
@@ -488,7 +488,7 @@ m filterExpression
segment.getFilteredIndexShardNames().clear();
// Check the segment exist in any of the pruned blocklets.
for (ExtendedBlocklet blocklet : prunedBlocklets) {
- if (blocklet.getSegmentId().equals(segment.getSegmentNo())) {
+ if (blocklet.getSegmentId().equals(segment.toString())) {
found = true;
// Set the pruned index file to the segment for further pruning.
String shardName = CarbonTablePath.getShardName(blocklet.getFilePath());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 4feb044..b549b16 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -420,6 +420,7 @@ public class CarbonTableInputFormat<T> extends CarbonInputFormat<T> {
List<Segment> invalidSegments = new ArrayList<>();
List<UpdateVO> invalidTimestampsList = new ArrayList<>();
+
try {
carbonTable = getOrCreateCarbonTable(job.getConfiguration());
ReadCommittedScope readCommittedScope =
@@ -427,7 +428,9 @@ public class CarbonTableInputFormat<T> extends CarbonInputFormat<T> {
this.readCommittedScope = readCommittedScope;
List<Segment> segmentList = new ArrayList<>();
- segmentList.add(new Segment(targetSegment, null, readCommittedScope));
+ Segment segment = Segment.getSegment(targetSegment, carbonTable.getTablePath());
+ segmentList.add(
+ new Segment(segment.getSegmentNo(), segment.getSegmentFileName(), readCommittedScope));
setSegmentsToAccess(job.getConfiguration(), segmentList);
// process and resolve the expression
@@ -599,7 +602,8 @@ public class CarbonTableInputFormat<T> extends CarbonInputFormat<T> {
long rowCount = blocklet.getDetailInfo().getRowCount();
- String key = CarbonUpdateUtil.getSegmentBlockNameKey(blocklet.getSegmentId(), blockName);
+ String segmentId = Segment.toSegment(blocklet.getSegmentId()).getSegmentNo();
+ String key = CarbonUpdateUtil.getSegmentBlockNameKey(segmentId, blockName);
// if block is invalid then dont add the count
SegmentUpdateDetails details = updateStatusManager.getDetailsForABlock(key);
@@ -608,11 +612,11 @@ public class CarbonTableInputFormat<T> extends CarbonInputFormat<T> {
Long blockCount = blockRowCountMapping.get(key);
if (blockCount == null) {
blockCount = 0L;
- Long count = segmentAndBlockCountMapping.get(blocklet.getSegmentId());
+ Long count = segmentAndBlockCountMapping.get(segmentId);
if (count == null) {
count = 0L;
}
- segmentAndBlockCountMapping.put(blocklet.getSegmentId(), count + 1);
+ segmentAndBlockCountMapping.put(segmentId, count + 1);
}
blockCount += rowCount;
blockRowCountMapping.put(key, blockCount);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
index 79458f5..f4d7034 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
@@ -3301,7 +3301,12 @@ test("IUD-01-01-02_023-67", Include) {
//Delete the uniqdata table
test("IUD-01-01-02_023-68", Include) {
sql(s"""use default""").collect
- sql(s"""delete from table uniqdata where segment.id IN(0)""").collect
+ try {
+ sql(s"""delete from table uniqdata where segment.id IN(0)""").collect
+ } catch {
+ case e: Exception =>
+ // ignore as data is already deleted in segment 0
+ }
checkAnswer(s"""select DOJ from uniqdata where CUST_ID=9001""",
Seq(Row(Timestamp.valueOf("2012-01-12 03:14:05.0"))), "DataLoadingIUDTestCase_IUD-01-01-02_023-68")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
index b027ce2..99a537a 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
@@ -18,12 +18,16 @@
package org.apache.carbondata.cluster.sdv.generated
+import scala.collection.JavaConverters._
+
import org.apache.spark.sql.common.util._
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
import org.apache.carbondata.core.writer.CarbonIndexFileMergeWriter
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -93,35 +97,36 @@ class MergeIndexTestCase extends QueryTest with BeforeAndAfterAll {
val table = CarbonMetadata.getInstance().getCarbonTable("default","carbon_automation_nonmerge")
new CarbonIndexFileMergeWriter(table).mergeCarbonIndexFilesOfSegment("0.1", table.getTablePath, false)
assert(getIndexFileCount("default", "carbon_automation_nonmerge", "0.1") == 0)
- assert(getMergedIndexFileCount("default", "carbon_automation_nonmerge", "0.1") == 1)
+ assert(getIndexFileCount("default", "carbon_automation_nonmerge", "0.1", true) >= 1)
checkAnswer(sql("""Select count(*) from carbon_automation_nonmerge"""), rows)
}
- private def getIndexFileCount(dbName: String, tableName: String, segment: String): Int = {
- getFileCount(dbName, tableName, segment, CarbonTablePath.INDEX_FILE_EXT)
- }
-
- private def getMergedIndexFileCount(dbName: String, tableName: String, segment: String): Int = {
- getFileCount(dbName, tableName, segment, CarbonTablePath.MERGE_INDEX_FILE_EXT)
- }
-
- private def getFileCount(dbName: String,
- tableName: String,
- segment: String,
- suffix: String): Int = {
+ private def getIndexFileCount(dbName: String, tableName: String, segmentNo: String, mergeIndexCount: Boolean = false): Int = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(dbName, tableName)
- val identifier = carbonTable.getAbsoluteTableIdentifier
- val path = CarbonTablePath
- .getSegmentPath(identifier.getTablePath, segment)
- val carbonFiles = FileFactory.getCarbonFile(path).listFiles(new CarbonFileFilter {
- override def accept(file: CarbonFile): Boolean = {
- file.getName.endsWith(suffix)
- }
- })
- if (carbonFiles != null) {
- carbonFiles.length
+ val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentNo)
+ if (FileFactory.isFileExist(segmentDir)) {
+ val map = new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir)
+ map.asScala.map { f =>
+ if (f._2 == null) {
+ 1
+ } else {
+ if (mergeIndexCount) 1 else 0
+ }
+ }.sum
} else {
- 0
+ val segment = Segment.getSegment(segmentNo, carbonTable.getTablePath)
+ if (segment != null) {
+ val store = new SegmentFileStore(carbonTable.getTablePath, segment.getSegmentFileName)
+ store.getSegmentFile.getLocationMap.values().asScala.map { f =>
+ if (f.getMergeFileName == null) {
+ f.getFiles.size()
+ } else {
+ if (mergeIndexCount) 1 else 0
+ }
+ }.sum
+ } else {
+ 0
+ }
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
index f64a349..6530ec0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -541,37 +541,51 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
}
test("test lucene fine grain data map with text-match limit") {
-
+ sql("DROP TABLE IF EXISTS datamap_test_limit")
+ sql(
+ """
+ | CREATE TABLE datamap_test_limit(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
sql(
s"""
- | CREATE DATAMAP dm ON TABLE datamap_test
+ | CREATE DATAMAP dm ON TABLE datamap_test_limit
| USING 'lucene'
| DMProperties('INDEX_COLUMNS'='name , city')
""".stripMargin)
- checkAnswer(sql("select count(*) from datamap_test where TEXT_MATCH_WITH_LIMIT('name:n10*',10)"),Seq(Row(10)))
- checkAnswer(sql("select count(*) from datamap_test where TEXT_MATCH_WITH_LIMIT('name:n10*',50)"),Seq(Row(50)))
- sql("drop datamap dm on table datamap_test")
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_limit OPTIONS('header'='false')")
+ checkAnswer(sql("select count(*) from datamap_test_limit where TEXT_MATCH_WITH_LIMIT('name:n10*',10)"),Seq(Row(10)))
+ checkAnswer(sql("select count(*) from datamap_test_limit where TEXT_MATCH_WITH_LIMIT('name:n10*',50)"),Seq(Row(50)))
+ sql("drop datamap dm on table datamap_test_limit")
}
test("test lucene fine grain data map with InsertOverwrite") {
+ sql("DROP TABLE IF EXISTS datamap_test_overwrite")
+ sql(
+ """
+ | CREATE TABLE datamap_test_overwrite(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
sql(
s"""
- | CREATE DATAMAP dm ON TABLE datamap_test
+ | CREATE DATAMAP dm ON TABLE datamap_test_overwrite
| USING 'lucene'
| DMProperties('INDEX_COLUMNS'='name , city')
""".stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_overwrite OPTIONS('header'='false')")
sql(
"""
| CREATE TABLE table1(id INT, name STRING, city STRING, age INT)
| STORED BY 'carbondata'
| TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
""".stripMargin)
- sql("INSERT OVERWRITE TABLE table1 select * from datamap_test where TEXT_MATCH('name:n*')")
+ sql("INSERT OVERWRITE TABLE table1 select *from datamap_test_overwrite where TEXT_MATCH('name:n*')")
checkAnswer(sql("select count(*) from table1"),Seq(Row(10000)))
- sql("drop datamap dm on table datamap_test")
- sql("drop table table1")
+ sql("drop datamap dm on table datamap_test_overwrite")
}
test("explain query with lucene datamap") {
@@ -715,7 +729,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
sql(s"SELECT * FROM datamap_test5 WHERE city='c020'"))
sql("DROP TABLE IF EXISTS datamap_test5")
}
-
+
test("test text_match on normal table") {
sql("DROP TABLE IF EXISTS table1")
sql(
@@ -731,7 +745,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
assert(msg.getCause.getMessage.contains("TEXT_MATCH is not supported on table"))
sql("DROP TABLE table1")
}
-
+
test("test lucene with flush_cache as true") {
sql("DROP TABLE IF EXISTS datamap_test_table")
sql(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
index 43b215e..688928f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
@@ -26,10 +26,11 @@ import org.scalatest.BeforeAndAfterEach
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
import org.apache.spark.sql.test.util.QueryTest
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
+import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.util.CarbonProperties
class TestLoadDataGeneral extends QueryTest with BeforeAndAfterEach {
@@ -53,10 +54,8 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterEach {
val fileType: FileFactory.FileType = FileFactory.getFileType(partitionPath)
val carbonFile = FileFactory.getCarbonFile(partitionPath, fileType)
val segments: ArrayBuffer[String] = ArrayBuffer()
- carbonFile.listFiles.foreach { file =>
- segments += CarbonTablePath.DataFileUtil.getSegmentId(file.getAbsolutePath + "/dummy")
- }
- segments.contains(segmentId)
+ val segment = Segment.getSegment(segmentId, carbonTable.getAbsoluteTableIdentifier.getTablePath)
+ segment != null
}
test("test data loading CSV file") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
index 4860b32..8487b9e 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
@@ -16,17 +16,13 @@
*/
package org.apache.carbondata.spark.testsuite.allqueries
-import java.io.File
-
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.CarbonMetadata
-import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
var timeStampPropOrig: String = _
@@ -227,11 +223,7 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("insert overwrite table CarbonOverwrite select * from THive")
sql("insert overwrite table HiveOverwrite select * from THive")
checkAnswer(sql("select count(*) from CarbonOverwrite"), sql("select count(*) from HiveOverwrite"))
- val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "carbonoverwrite")
- val partitionPath = CarbonTablePath.getPartitionDir(carbonTable.getAbsoluteTableIdentifier.getTablePath)
- val folder = new File(partitionPath)
- assert(folder.isDirectory)
- assert(folder.list().length == 1)
+ assert(checkSegment("CarbonOverwrite"))
}
test("Load overwrite") {
@@ -249,12 +241,7 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("LOAD DATA INPATH '" + resourcesPath + "/100_olap.csv' overwrite INTO table TCarbonSourceOverwrite options ('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVe
rsion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointDescription,gamePointId,contractNumber')")
sql(s"LOAD DATA local INPATH '$resourcesPath/100_olap.csv' overwrite INTO TABLE HiveOverwrite")
checkAnswer(sql("select count(*) from TCarbonSourceOverwrite"), sql("select count(*) from HiveOverwrite"))
- val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "tcarbonsourceoverwrite")
- val partitionPath = CarbonTablePath.getPartitionDir(carbonTable.getAbsoluteTableIdentifier.getTablePath)
- val folder = new File(partitionPath)
-
- assert(folder.isDirectory)
- assert(folder.list().length == 1)
+ assert(checkSegment("TCarbonSourceOverwrite"))
}
test("Load overwrite fail handle") {
@@ -379,15 +366,9 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
private def checkSegment(tableName: String) : Boolean ={
- val storePath_t1 = s"$storeLocation/${tableName.toLowerCase()}/Fact/Part0"
- val carbonFile_t1: CarbonFile = FileFactory
- .getCarbonFile(storePath_t1, FileFactory.getFileType(storePath_t1))
- var exists: Boolean = carbonFile_t1.exists()
- if (exists) {
- val listFiles: Array[CarbonFile] = carbonFile_t1.listFiles()
- exists = listFiles.size > 0
- }
- exists
+ val storePath_t1 = s"$storeLocation/${tableName.toLowerCase()}"
+ val detailses = SegmentStatusManager.readTableStatusFile(CarbonTablePath.getTableStatusFilePath(storePath_t1))
+ detailses.map(_.getSegmentStatus == SegmentStatus.SUCCESS).exists(f => f)
}
test("test show segments after clean files for insert overwrite") {
[35/50] [abbrv] carbondata git commit: [CARBONDATA-1787] Updated
data-management-on-carbondata.md for GLOBAL_SORT_PARTITIONS
Posted by ja...@apache.org.
[CARBONDATA-1787] Updated data-management-on-carbondata.md for GLOBAL_SORT_PARTITIONS
This closes #1668
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ca466d9f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ca466d9f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ca466d9f
Branch: refs/heads/carbonstore
Commit: ca466d9f4b07db1a088dc62cebaf6b4733c28a9b
Parents: dc4f87b
Author: vandana <va...@gmail.com>
Authored: Fri Dec 15 18:16:05 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Tue Jun 19 00:34:39 2018 +0800
----------------------------------------------------------------------
docs/data-management-on-carbondata.md | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ca466d9f/docs/data-management-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/data-management-on-carbondata.md b/docs/data-management-on-carbondata.md
index 3326e9b..7e171aa 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -560,6 +560,16 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
'BAD_RECORDS_ACTION'='REDIRECT','IS_EMPTY_DATA_BAD_RECORD'='false')
```
+ - **GLOBAL_SORT_PARTITIONS:** If the SORT_SCOPE is defined as GLOBAL_SORT, then user can specify the number of partitions to use while shuffling data for sort using GLOBAL_SORT_PARTITIONS. If it is not configured, or configured less than 1, then it uses the number of map task as reduce task. It is recommended that each reduce task deal with 512MB-1GB data.
+
+ ```
+ OPTIONS('GLOBAL_SORT_PARTITIONS'='2')
+ ```
+
+ NOTE:
+ * GLOBAL_SORT_PARTITIONS should be Integer type, the range is [1,Integer.MaxValue].
+ * It is only used when the SORT_SCOPE is GLOBAL_SORT.
+
### INSERT DATA INTO CARBONDATA TABLE
This command inserts data into a CarbonData table, it is defined as a combination of two queries Insert and Select query respectively.
[20/50] [abbrv] carbondata git commit: [CARBONDATA-2578] fixed memory
leak inside CarbonReader and handled failure case for creation of multi
reader for non-transactional table
Posted by ja...@apache.org.
[CARBONDATA-2578] fixed memory leak inside CarbonReader and handled failure case for creation of
multi reader for non-transactional table
Issue :
CarbonIterator inside CarbonRecordReader was keeping reference of RowBatch and it is not being
closed inside CarbonRecordReader. sort_column with measure was considering all the dimension
column along with given column. if creation of one CarbonReader for non transactional table
is failed then we are not able to create another CarbonReader.
Solution :
close() called inside hasNext() to clear the previous iterator before iterating over the next CarbonReader..
if sortcolumn is not empty and sortcolumnsList contains the fields check is added to finally in sortcolumn props.
Clear the datamap in catch block, if creation of CarbonReader is failed.
This closes #2362
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/19312ab5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/19312ab5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/19312ab5
Branch: refs/heads/carbonstore
Commit: 19312ab5b35654e89f04ec79881d928c859f8a07
Parents: 290ef5a
Author: rahul <ra...@knoldus.in>
Authored: Tue Jun 5 13:09:36 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Tue Jun 12 17:01:53 2018 +0530
----------------------------------------------------------------------
.../createTable/TestNonTransactionalCarbonTable.scala | 7 ++++---
.../java/org/apache/carbondata/sdk/file/CarbonReader.java | 2 ++
.../org/apache/carbondata/sdk/file/CarbonWriterBuilder.java | 5 -----
3 files changed, 6 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index b275bb8..805fc71 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -368,8 +368,9 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
- checkExistence(sql("describe formatted sdkOutputTable"), true, "age")
-
+ checkExistence(sql("describe formatted sdkOutputTable"), true, "SORT_COLUMNS age")
+ checkExistence(sql("describe formatted sdkOutputTable"), false, "SORT_COLUMNS name,age")
+ checkExistence(sql("describe formatted sdkOutputTable"), false, "SORT_COLUMNS age,name")
buildTestDataSingleFile()
assert(new File(writerPath).exists())
sql("DROP TABLE IF EXISTS sdkOutputTable")
@@ -402,7 +403,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
intercept[RuntimeException] {
buildTestDataWithSortColumns(List(""))
}
-
+
assert(!(new File(writerPath).exists()))
cleanTestData()
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
index 9af710f..be809e6 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
@@ -74,6 +74,8 @@ public class CarbonReader<T> {
return false;
} else {
index++;
+ // current reader is closed
+ currentReader.close();
currentReader = readers.get(index);
return currentReader.nextKeyValue();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index bd64568..0f670fe 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -484,11 +484,6 @@ public class CarbonWriterBuilder {
if (isSortColumn > -1) {
columnSchema.setSortColumn(true);
sortColumnsSchemaList[isSortColumn] = columnSchema;
- } else if (!sortColumnsList.isEmpty() && columnSchema.isDimensionColumn()
- && columnSchema.getNumberOfChild() < 1) {
- columnSchema.setSortColumn(true);
- sortColumnsSchemaList[i] = columnSchema;
- i++;
}
}
}
[45/50] [abbrv] carbondata git commit: [CARBONDATA-2504][STREAM]
Support StreamSQL for streaming job
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/store/sdk/src/main/java/org/apache/carbondata/store/LocalCarbonStore.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/store/LocalCarbonStore.java b/store/sdk/src/main/java/org/apache/carbondata/store/LocalCarbonStore.java
index 9e338e7..daa1447 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/store/LocalCarbonStore.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/store/LocalCarbonStore.java
@@ -65,7 +65,7 @@ class LocalCarbonStore extends MetaCachedCarbonStore {
Objects.requireNonNull(projectColumns);
CarbonTable table = getTable(path);
- if (table.isStreamingTable() || table.isHivePartitionTable()) {
+ if (table.isStreamingSink() || table.isHivePartitionTable()) {
throw new UnsupportedOperationException("streaming and partition table is not supported");
}
// TODO: use InputFormat to prune data and read data
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala
index 1696fdc..5a888ef 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala
@@ -17,7 +17,6 @@
package org.apache.carbondata.streaming.parser
-import java.nio.charset.Charset
import java.text.SimpleDateFormat
import org.apache.hadoop.conf.Configuration
[05/50] [abbrv] carbondata git commit: [CARBONDATA-2355] Support run
SQL on carbondata files directly
Posted by ja...@apache.org.
[CARBONDATA-2355] Support run SQL on carbondata files directly
Support run SQL on carbondata files directly
This closes #2181
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9469e6bd
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9469e6bd
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9469e6bd
Branch: refs/heads/carbonstore
Commit: 9469e6bd4da5c75ba836fb550112cec01f666544
Parents: 4d22ddc
Author: xubo245 <60...@qq.com>
Authored: Wed Apr 18 17:34:12 2018 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Fri Jun 1 18:01:33 2018 +0800
----------------------------------------------------------------------
docs/sdk-guide.md | 7 ++
.../carbondata/examples/DirectSQLExample.scala | 100 +++++++++++++++++++
.../carbondata/examples/S3UsingSDkExample.scala | 2 +-
...FileInputFormatWithExternalCarbonTable.scala | 2 +-
...tCreateTableUsingSparkCarbonFileFormat.scala | 30 +++++-
.../TestNonTransactionalCarbonTable.scala | 2 +-
...ransactionalCarbonTableWithComplexType.scala | 2 +-
...tSparkCarbonFileFormatWithSparkSession.scala | 2 +-
.../datasources/SparkCarbonFileFormat.scala | 26 ++++-
9 files changed, 164 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9469e6bd/docs/sdk-guide.md
----------------------------------------------------------------------
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index 360516a..ec70919 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -128,7 +128,14 @@ Each of SQL data types are mapped into data types of SDK. Following are the mapp
| STRING | DataTypes.STRING |
| DECIMAL | DataTypes.createDecimalType(precision, scale) |
+## Run SQL on files directly
+Instead of creating table and query it, you can also query that file directly with SQL.
+### Example
+```
+SELECT * FROM carbonfile.`$Path`
+```
+Find example code at [DirectSQLExample](https://github.com/apache/carbondata/blob/master/examples/spark2/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala) in the CarbonData repo.
## API List
### Class org.apache.carbondata.sdk.file.CarbonWriterBuilder
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9469e6bd/examples/spark2/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala
new file mode 100644
index 0000000..a011d80
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.examples
+
+import java.io.File
+
+import org.apache.commons.io.FileUtils
+
+import org.apache.carbondata.core.metadata.datatype.DataTypes
+import org.apache.carbondata.examples.util.ExampleUtils
+import org.apache.carbondata.sdk.file.{CarbonWriter, Field, Schema}
+
+/**
+ * Running SQL on carbon files directly
+ * No need to create table first
+ * TODO: support more than one carbon file
+ */
+object DirectSQLExample {
+
+ // prepare SDK writer output
+ def buildTestData(
+ path: String,
+ num: Int = 3,
+ persistSchema: Boolean = false): Any = {
+
+ // getCanonicalPath gives path with \, but the code expects /.
+ val writerPath = path.replace("\\", "/");
+
+ val fields: Array[Field] = new Array[Field](3)
+ fields(0) = new Field("name", DataTypes.STRING)
+ fields(1) = new Field("age", DataTypes.INT)
+ fields(2) = new Field("height", DataTypes.DOUBLE)
+
+ try {
+ val builder = CarbonWriter
+ .builder()
+ .outputPath(writerPath)
+ .isTransactionalTable(true)
+ .uniqueIdentifier(System.currentTimeMillis)
+ .withBlockSize(2)
+ if (persistSchema) {
+ builder.persistSchemaFile(true)
+ }
+ val writer = builder.buildWriterForCSVInput(new Schema(fields))
+ var i = 0
+ while (i < num) {
+ writer.write(Array[String]("robot" + i, String.valueOf(i), String.valueOf(i.toDouble / 2)))
+ i += 1
+ }
+ writer.close()
+ } catch {
+ case e: Exception => throw e
+ }
+ }
+
+ def cleanTestData(path: String): Unit = {
+ FileUtils.deleteDirectory(new File(path))
+ }
+
+ // scalastyle:off
+ def main(args: Array[String]) {
+ val carbonSession = ExampleUtils.createCarbonSession("DirectSQLExample")
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val path = s"$rootPath/examples/spark2/target/carbonFile/"
+
+ import carbonSession._
+ // 1. generate data file
+ cleanTestData(path)
+ buildTestData(path, 20)
+ val readPath = path + "Fact/Part0/Segment_null"
+
+ println("Running SQL on carbon files directly")
+ try {
+ // 2. run queries directly, no need to create table first
+ sql(s"""select * FROM carbonfile.`$readPath` limit 10""".stripMargin).show()
+ } catch {
+ case e: Exception => throw e
+ } finally {
+ // 3.delete data files
+ cleanTestData(path)
+ }
+ }
+ // scalastyle:on
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9469e6bd/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala
index 022b28e..1795960 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala
@@ -36,7 +36,7 @@ object S3UsingSDKExample {
num: Int = 3,
persistSchema: Boolean = false): Any = {
- // getCanonicalPath gives path with \, so code expects /. Need to handle in code ?
+ // getCanonicalPath gives path with \, but the code expects /.
val writerPath = path.replace("\\", "/");
val fields: Array[Field] = new Array[Field](3)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9469e6bd/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCarbonFileInputFormatWithExternalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCarbonFileInputFormatWithExternalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCarbonFileInputFormatWithExternalCarbonTable.scala
index 019b915..e6d39d3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCarbonFileInputFormatWithExternalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCarbonFileInputFormatWithExternalCarbonTable.scala
@@ -38,7 +38,7 @@ class TestCarbonFileInputFormatWithExternalCarbonTable extends QueryTest with Be
"../." +
"./src/test/resources/SparkCarbonFileFormat/WriterOutput/")
.getCanonicalPath
- //getCanonicalPath gives path with \, so code expects /. Need to handle in code ?
+ //getCanonicalPath gives path with \, but the code expects /.
writerPath = writerPath.replace("\\", "/");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9469e6bd/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableUsingSparkCarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableUsingSparkCarbonFileFormat.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableUsingSparkCarbonFileFormat.scala
index 66be8e4..211bc8c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableUsingSparkCarbonFileFormat.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableUsingSparkCarbonFileFormat.scala
@@ -46,7 +46,7 @@ class TestCreateTableUsingSparkCarbonFileFormat extends QueryTest with BeforeAnd
"../." +
"./src/test/resources/SparkCarbonFileFormat/WriterOutput/")
.getCanonicalPath
- //getCanonicalPath gives path with \, so code expects /. Need to handle in code ?
+ //getCanonicalPath gives path with \, but the code expects /.
writerPath = writerPath.replace("\\", "/");
val filePath = writerPath + "/Fact/Part0/Segment_null/"
@@ -153,6 +153,34 @@ class TestCreateTableUsingSparkCarbonFileFormat extends QueryTest with BeforeAnd
cleanTestData()
}
+ test("Running SQL directly and read carbondata files (sdk Writer Output) using the SparkCarbonFileFormat ") {
+ buildTestData(false)
+ assert(new File(filePath).exists())
+ sql("DROP TABLE IF EXISTS sdkOutputTable")
+
+ //data source file format
+ if (sqlContext.sparkContext.version.startsWith("2.1")) {
+ //data source file format
+ sql(s"""CREATE TABLE sdkOutputTable USING carbonfile OPTIONS (PATH '$filePath') """)
+ } else if (sqlContext.sparkContext.version.startsWith("2.2")) {
+ //data source file format
+ sql(
+ s"""CREATE TABLE sdkOutputTable USING carbonfile LOCATION
+ |'$filePath' """.stripMargin)
+ } else {
+ // TO DO
+ }
+
+ val directSQL = sql(s"""select * FROM carbonfile.`$filePath`""".stripMargin)
+ directSQL.show(false)
+ checkAnswer(sql("select * from sdkOutputTable"), directSQL)
+
+ sql("DROP TABLE sdkOutputTable")
+ // drop table should not delete the files
+ assert(new File(filePath).exists())
+ cleanTestData()
+ }
+
test("should not allow to alter datasource carbontable ") {
buildTestData(false)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9469e6bd/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index 61b37d5..0083733 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -55,7 +55,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
"../." +
"./target/SparkCarbonFileFormat/WriterOutput/")
.getCanonicalPath
- //getCanonicalPath gives path with \, so code expects /. Need to handle in code ?
+ //getCanonicalPath gives path with \, but the code expects /.
writerPath = writerPath.replace("\\", "/")
def buildTestDataSingleFile(): Any = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9469e6bd/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala
index d4de428..19aaf72 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala
@@ -39,7 +39,7 @@ class TestNonTransactionalCarbonTableWithComplexType extends QueryTest with Befo
"../." +
"./src/test/resources/SparkCarbonFileFormat/WriterOutput/")
.getCanonicalPath
- //getCanonicalPath gives path with \, so code expects /. Need to handle in code ?
+ //getCanonicalPath gives path with \, but the code expects /.
writerPath = writerPath.replace("\\", "/")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9469e6bd/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestSparkCarbonFileFormatWithSparkSession.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestSparkCarbonFileFormatWithSparkSession.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestSparkCarbonFileFormatWithSparkSession.scala
index 54b23a5..79b64ae 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestSparkCarbonFileFormatWithSparkSession.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestSparkCarbonFileFormatWithSparkSession.scala
@@ -36,7 +36,7 @@ object TestSparkCarbonFileFormatWithSparkSession {
"../." +
"./src/test/resources/SparkCarbonFileFormat/WriterOutput/")
.getCanonicalPath
- //getCanonicalPath gives path with \, so code expects /. Need to handle in code ?
+ //getCanonicalPath gives path with \, but the code expects /.
writerPath = writerPath.replace("\\", "/");
val filePath = writerPath + "/Fact/Part0/Segment_null/"
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9469e6bd/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonFileFormat.scala
index 934f5c7..697eec5 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonFileFormat.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.datasources
import java.net.URI
+import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import org.apache.hadoop.conf.Configuration
@@ -68,8 +69,23 @@ class SparkCarbonFileFormat extends FileFormat
override def inferSchema(sparkSession: SparkSession,
options: Map[String, String],
files: Seq[FileStatus]): Option[StructType] = {
- val filePaths = CarbonUtil.getFilePathExternalFilePath(
- options.get("path").get)
+ val filePaths = if (options.isEmpty) {
+ val carbondataFiles = files.seq.filter { each =>
+ if (each.isFile) {
+ each.getPath.getName.contains(".carbondata")
+ } else {
+ false
+ }
+ }
+
+ carbondataFiles.map { each =>
+ each.getPath.toString
+ }.toList.asJava
+ } else {
+ CarbonUtil.getFilePathExternalFilePath(
+ options.get("path").get)
+ }
+
if (filePaths.size() == 0) {
throw new SparkException("CarbonData file is not present in the location mentioned in DDL")
}
@@ -193,7 +209,11 @@ class SparkCarbonFileFormat extends FileFormat
val fileSplit =
new FileSplit(new Path(new URI(file.filePath)), file.start, file.length, Array.empty)
- val path: String = options.get("path").get
+ val path: String = if (options.isEmpty) {
+ file.filePath
+ } else {
+ options.get("path").get
+ }
val endindex: Int = path.indexOf("Fact") - 1
val tablePath = path.substring(0, endindex)
lazy val identifier: AbsoluteTableIdentifier = AbsoluteTableIdentifier.from(
[48/50] [abbrv] carbondata git commit:
[CARBONDATA-2509][CARBONDATA-2510][CARBONDATA-2511][32K] Add validate for
long string columns
Posted by ja...@apache.org.
[CARBONDATA-2509][CARBONDATA-2510][CARBONDATA-2511][32K] Add validate for long string columns
Add validate for long string columns
1. long string columns cannot be sort_columns
2. long string columns cannot be dictionary include
3. long string columns cannot be dictionary exclude
4. long string columns can only be string columns
This closes #2380
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/218a8deb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/218a8deb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/218a8deb
Branch: refs/heads/carbonstore
Commit: 218a8deb614e0dc160ab8c7e38c71d80711eb1a7
Parents: 091a28b
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Wed Jun 20 18:55:04 2018 +0800
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Thu Jun 21 12:00:34 2018 +0530
----------------------------------------------------------------------
.../VarcharDataTypesBasicTestCase.scala | 57 ++++++++++++++++++++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 11 +++-
2 files changed, 67 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/218a8deb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
index 419b306..9ea3f1f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
@@ -74,6 +74,63 @@ class VarcharDataTypesBasicTestCase extends QueryTest with BeforeAndAfterEach wi
sql(s"drop table if exists $longStringTable")
}
+ test("long string columns cannot be dictionary include") {
+ val exceptionCaught = intercept[Exception] {
+ sql(
+ s"""
+ | CREATE TABLE if not exists $longStringTable(
+ | id INT, name STRING, description STRING, address STRING, note STRING
+ | ) STORED BY 'carbondata'
+ | TBLPROPERTIES('LONG_STRING_COLUMNS'='address, note', 'dictionary_include'='address')
+ |""".
+ stripMargin)
+ }
+ assert(exceptionCaught.getMessage.contains("DICTIONARY_INCLUDE is unsupported for long string datatype column: address"))
+ }
+
+ test("long string columns cannot be dictionay exclude") {
+ val exceptionCaught = intercept[Exception] {
+ sql(
+ s"""
+ | CREATE TABLE if not exists $longStringTable(
+ | id INT, name STRING, description STRING, address STRING, note STRING
+ | ) STORED BY 'carbondata'
+ | TBLPROPERTIES('LONG_STRING_COLUMNS'='address, note', 'dictionary_exclude'='address')
+ |""".
+ stripMargin)
+ }
+ assert(exceptionCaught.getMessage.contains("DICTIONARY_EXCLUDE is unsupported for long string datatype column: address"))
+ }
+
+ test("long string columns cannot be sort_columns") {
+ val exceptionCaught = intercept[Exception] {
+ sql(
+ s"""
+ | CREATE TABLE if not exists $longStringTable(
+ | id INT, name STRING, description STRING, address STRING, note STRING
+ | ) STORED BY 'carbondata'
+ | TBLPROPERTIES('LONG_STRING_COLUMNS'='name, note', 'SORT_COLUMNS'='name, address')
+ |""".
+ stripMargin)
+ }
+ assert(exceptionCaught.getMessage.contains("sort_columns is unsupported for long string datatype column: name"))
+ }
+
+ test("long string columns can only be string columns") {
+ val exceptionCaught = intercept[Exception] {
+ sql(
+ s"""
+ | CREATE TABLE if not exists $longStringTable(
+ | id INT, name STRING, description STRING, address STRING, note STRING
+ | ) STORED BY 'carbondata'
+ | TBLPROPERTIES('LONG_STRING_COLUMNS'='id, note')
+ |""".
+ stripMargin)
+ }
+ assert(exceptionCaught.getMessage.contains("long_string_columns: id"))
+ assert(exceptionCaught.getMessage.contains("its data type is not string"))
+ }
+
private def prepareTable(): Unit = {
sql(
s"""
http://git-wip-us.apache.org/repos/asf/carbondata/blob/218a8deb/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 350fc36..7c3cf65 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -752,7 +752,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
if (varcharCols.exists(x => x.equalsIgnoreCase(column))) {
throw new MalformedCarbonCommandException(
- s"sort_columns is unsupported for long string datatype column $column")
+ s"sort_columns is unsupported for long string datatype column: $column")
}
}
}
@@ -791,6 +791,10 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
val errorMsg = "DICTIONARY_EXCLUDE is unsupported for " + dataType.toLowerCase() +
" data type column: " + dictExcludeCol
throw new MalformedCarbonCommandException(errorMsg)
+ } else if (varcharCols.exists(x => x.equalsIgnoreCase(dictExcludeCol))) {
+ throw new MalformedCarbonCommandException(
+ "DICTIONARY_EXCLUDE is unsupported for long string datatype column: " +
+ dictExcludeCol)
}
}
}
@@ -805,6 +809,11 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
" does not exist in table. Please check create table statement."
throw new MalformedCarbonCommandException(errormsg)
}
+ if (varcharCols.exists(x => x.equalsIgnoreCase(distIncludeCol.trim))) {
+ throw new MalformedCarbonCommandException(
+ "DICTIONARY_INCLUDE is unsupported for long string datatype column: " +
+ distIncludeCol.trim)
+ }
}
}
[27/50] [abbrv] carbondata git commit: [CARBONDATA-2593] Add an
option 'carbon.insert.storage.level' to support configuring the storage level
when insert into data with 'carbon.insert.persist.enable'='true'
Posted by ja...@apache.org.
[CARBONDATA-2593] Add an option 'carbon.insert.storage.level' to support configuring the storage level when insert into data with 'carbon.insert.persist.enable'='true'
When insert into data with 'carbon.insert.persist.enable'='true', the storage level of dataset is 'MEMORY_AND_DISK',
it should support configuring the storage level to correspond to different environment.
This closes #2373
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/181f0ac9
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/181f0ac9
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/181f0ac9
Branch: refs/heads/carbonstore
Commit: 181f0ac9bed6ff7d83268f6c058aee943b348ddc
Parents: f0c8834
Author: Zhang Zhichao <44...@qq.com>
Authored: Thu Jun 14 14:48:47 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Jun 16 03:32:36 2018 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 25 ++++++++++++++++++++
.../carbondata/core/util/CarbonProperties.java | 18 ++++++++++++++
docs/configuration-parameters.md | 4 ++++
.../management/CarbonInsertIntoCommand.scala | 5 ++--
4 files changed, 50 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181f0ac9/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index c7281dd..19ff494 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -866,6 +866,7 @@ public final class CarbonCommonConstants {
* to run load and insert queries on source table concurrently then user can enable this flag
*/
@CarbonProperty
+ @InterfaceStability.Evolving
public static final String CARBON_INSERT_PERSIST_ENABLED = "carbon.insert.persist.enable";
/**
@@ -875,6 +876,27 @@ public final class CarbonCommonConstants {
public static final String CARBON_INSERT_PERSIST_ENABLED_DEFAULT = "false";
/**
+ * Which storage level to persist dataset when insert into data
+ * with 'carbon.insert.persist.enable'='true'
+ */
+ @CarbonProperty
+ @InterfaceStability.Evolving
+ public static final String CARBON_INSERT_STORAGE_LEVEL =
+ "carbon.insert.storage.level";
+
+ /**
+ * The default value(MEMORY_AND_DISK) is the same as the default storage level of Dataset.
+ * Unlike `RDD.cache()`, the default storage level is set to be `MEMORY_AND_DISK` because
+ * recomputing the in-memory columnar representation of the underlying table is expensive.
+ *
+ * if user's executor has less memory, set the CARBON_INSERT_STORAGE_LEVEL
+ * to MEMORY_AND_DISK_SER or other storage level to correspond to different environment.
+ * You can get more recommendations about storage level in spark website:
+ * http://spark.apache.org/docs/latest/rdd-programming-guide.html#rdd-persistence.
+ */
+ public static final String CARBON_INSERT_STORAGE_LEVEL_DEFAULT = "MEMORY_AND_DISK";
+
+ /**
* default name of data base
*/
public static final String DATABASE_DEFAULT_NAME = "default";
@@ -1094,6 +1116,7 @@ public final class CarbonCommonConstants {
* to determine to use the rdd persist or not.
*/
@CarbonProperty
+ @InterfaceStability.Evolving
public static final String isPersistEnabled = "carbon.update.persist.enable";
/**
@@ -1117,6 +1140,7 @@ public final class CarbonCommonConstants {
* with 'carbon.update.persist.enable'='true'
*/
@CarbonProperty
+ @InterfaceStability.Evolving
public static final String CARBON_UPDATE_STORAGE_LEVEL =
"carbon.update.storage.level";
@@ -1354,6 +1378,7 @@ public final class CarbonCommonConstants {
* Which storage level to persist rdd when sort_scope=global_sort
*/
@CarbonProperty
+ @InterfaceStability.Evolving
public static final String CARBON_GLOBAL_SORT_RDD_STORAGE_LEVEL =
"carbon.global.sort.rdd.storage.level";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181f0ac9/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 6eb7de6..b134a7c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -1581,4 +1581,22 @@ public final class CarbonProperties {
return defaultValue;
}
}
+
+ /**
+ * Return valid storage level for CARBON_INSERT_STORAGE_LEVEL
+ * @return String
+ */
+ public String getInsertIntoDatasetStorageLevel() {
+ String storageLevel = getProperty(CarbonCommonConstants.CARBON_INSERT_STORAGE_LEVEL,
+ CarbonCommonConstants.CARBON_INSERT_STORAGE_LEVEL_DEFAULT);
+ boolean validateStorageLevel = CarbonUtil.isValidStorageLevel(storageLevel);
+ if (!validateStorageLevel) {
+ LOGGER.warn("The " + CarbonCommonConstants.CARBON_INSERT_STORAGE_LEVEL
+ + " configuration value is invalid. It will use default storage level("
+ + CarbonCommonConstants.CARBON_INSERT_STORAGE_LEVEL_DEFAULT
+ + ") to persist dataset.");
+ storageLevel = CarbonCommonConstants.CARBON_INSERT_STORAGE_LEVEL_DEFAULT;
+ }
+ return storageLevel.toUpperCase();
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181f0ac9/docs/configuration-parameters.md
----------------------------------------------------------------------
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index 11cc6ea..f81959e 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -55,7 +55,11 @@ This section provides the details of all the configurations required for CarbonD
| carbon.max.driver.lru.cache.size | -1 | Max LRU cache size upto which data will be loaded at the driver side. This value is expressed in MB. Default value of -1 means there is no memory limit for caching. Only integer values greater than 0 are accepted. | |
| carbon.max.executor.lru.cache.size | -1 | Max LRU cache size upto which data will be loaded at the executor side. This value is expressed in MB. Default value of -1 means there is no memory limit for caching. Only integer values greater than 0 are accepted. If this parameter is not configured, then the carbon.max.driver.lru.cache.size value will be considered. | |
| carbon.merge.sort.prefetch | true | Enable prefetch of data during merge sort while reading data from sort temp files in data loading. | |
+| carbon.insert.persist.enable | false | Enabling this parameter considers persistent data. If we are executing insert into query from source table using select statement & loading the same source table concurrently, when select happens on source table during the data load, it gets new record for which dictionary is not generated, so there will be inconsistency. To avoid this condition we can persist the dataframe into MEMORY_AND_DISK(default value) and perform insert into operation. By default this value will be false because no need to persist the dataframe in all cases. If user wants to run load and insert queries on source table concurrently then user can enable this parameter. | |
+| carbon.insert.storage.level | MEMORY_AND_DISK | Which storage level to persist dataframe when 'carbon.insert.persist.enable'=true, if user's executor has less memory, set this parameter to 'MEMORY_AND_DISK_SER' or other storage level to correspond to different environment. [See detail](http://spark.apache.org/docs/latest/rdd-programming-guide.html#rdd-persistence). | |
| carbon.update.persist.enable | true | Enabling this parameter considers persistent data. Enabling this will reduce the execution time of UPDATE operation. | |
+| carbon.update.storage.level | MEMORY_AND_DISK | Which storage level to persist dataframe when 'carbon.update.persist.enable'=true, if user's executor has less memory, set this parameter to 'MEMORY_AND_DISK_SER' or other storage level to correspond to different environment. [See detail](http://spark.apache.org/docs/latest/rdd-programming-guide.html#rdd-persistence). | |
+| carbon.global.sort.rdd.storage.level | MEMORY_ONLY | Which storage level to persist rdd when loading data with 'sort_scope'='global_sort', if user's executor has less memory, set this parameter to 'MEMORY_AND_DISK_SER' or other storage level to correspond to different environment. [See detail](http://spark.apache.org/docs/latest/rdd-programming-guide.html#rdd-persistence). | |
| carbon.load.global.sort.partitions | 0 | The Number of partitions to use when shuffling data for sort. If user don't configurate or configurate it less than 1, it uses the number of map tasks as reduce tasks. In general, we recommend 2-3 tasks per CPU core in your cluster.
| carbon.options.bad.records.logger.enable | false | Whether to create logs with details about bad records. | |
| carbon.bad.records.action | FORCE | This property can have four types of actions for bad records FORCE, REDIRECT, IGNORE and FAIL. If set to FORCE then it auto-corrects the data by storing the bad records as NULL. If set to REDIRECT then bad records are written to the raw CSV instead of being loaded. If set to IGNORE then bad records are neither loaded nor written to the raw CSV. If set to FAIL then data loading fails if any bad records are found. | |
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181f0ac9/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
index 702f954..6c74ad2 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
@@ -53,8 +53,9 @@ case class CarbonInsertIntoCommand(
val df =
if (isPersistRequired) {
LOGGER.info("Persist enabled for Insert operation")
- Dataset.ofRows(sparkSession, child)
- .persist(StorageLevel.MEMORY_AND_DISK)
+ Dataset.ofRows(sparkSession, child).persist(
+ StorageLevel.fromString(
+ CarbonProperties.getInstance.getInsertIntoDatasetStorageLevel))
} else {
Dataset.ofRows(sparkSession, child)
}
[26/50] [abbrv] carbondata git commit: [CARBONDATA-2592][Integration]
Getting NoSuchMethod error due to aws sdk multple version jar conflicts
Posted by ja...@apache.org.
[CARBONDATA-2592][Integration] Getting NoSuchMethod error due to aws sdk multple version jar conflicts
## What changes were proposed in this pull request?
Currently in Carbon Spark2 project multiple dependency for the aws-sdk jar has been defined,this will create issue when we build carbon-assembly jars with
latest versions of hadoop/spark project. As part of hadoop-aws project, already aws-sdk 1.10.6 version jar will be fetched, since in the carbon-spark2 pom.xml
there is an explicit dependency defined/hardcoded for aws-sdk 1.7.4(old version) this can lead to conflicts while loading the class files. because of this problem
when we run any carbon examples passing carbon-assembly jars as the argument using spark-submit none of the testcases will work.
As a solution we can remove this dependency(aws-sdk 1.7.4) as already hadoop-aws dependency defined in the pom.xml of carbon-spark2 project
will fetch the latest aws-sdk jars.
## How was this patch tested?
After updating the pom, manually projects has been build and the use-case mentioned above is manually tested.
This closes #2364
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f0c88348
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f0c88348
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f0c88348
Branch: refs/heads/carbonstore
Commit: f0c88348a88f46b1cdaaa1e9bc00a3ab6111e5ec
Parents: f116352
Author: sujith71955 <su...@gmail.com>
Authored: Thu Jun 7 12:38:56 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Jun 15 12:30:54 2018 +0530
----------------------------------------------------------------------
integration/spark2/pom.xml | 19 -------------------
1 file changed, 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c88348/integration/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml
index 2bce694..9b9e71d 100644
--- a/integration/spark2/pom.xml
+++ b/integration/spark2/pom.xml
@@ -103,25 +103,6 @@
</exclusions>
</dependency>
<dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk</artifactId>
- <version>1.7.4</version>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
<version>0.9.0</version>
[04/50] [abbrv] carbondata git commit: [CARBONDATA-2529] Fixed S3
Issue for Hadoop 2.8.3
Posted by ja...@apache.org.
[CARBONDATA-2529] Fixed S3 Issue for Hadoop 2.8.3
This issue fixes the issue while loading the data with S3 as backend
This closes #2340
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4d22ddc9
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4d22ddc9
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4d22ddc9
Branch: refs/heads/carbonstore
Commit: 4d22ddc9d932891af7d3f6557a423d65969f1fd3
Parents: 5ad7009
Author: Bhavya <bh...@knoldus.com>
Authored: Thu May 24 21:17:58 2018 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Fri Jun 1 14:37:36 2018 +0800
----------------------------------------------------------------------
.../carbondata/core/datastore/filesystem/HDFSCarbonFile.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d22ddc9/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
index 4663ac5..fc5420d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
@@ -120,7 +120,8 @@ public class HDFSCarbonFile extends AbstractDFSCarbonFile {
((DistributedFileSystem) fs).rename(fileStatus.getPath(), new Path(changetoName),
org.apache.hadoop.fs.Options.Rename.OVERWRITE);
return true;
- } else if (fileStatus.getPath().toString().startsWith("s3n")) {
+ } else if (fileStatus.getPath().toString().startsWith("s3n")
+ || fileStatus.getPath().toString().startsWith("s3a")) {
fs.delete(new Path(changetoName), true);
return fs.rename(fileStatus.getPath(), new Path(changetoName));
} else {
[47/50] [abbrv] carbondata git commit: [CARBONDATA-2615][32K] Support
page size less than 32000 in CarbondataV3
Posted by ja...@apache.org.
[CARBONDATA-2615][32K] Support page size less than 32000 in CarbondataV3
Since we support super long string, if it is long enough, a column page
with 32000 rows will exceed 2GB, so we support a page less than 32000
rows.
This closes #2383
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/091a28bf
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/091a28bf
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/091a28bf
Branch: refs/heads/carbonstore
Commit: 091a28bf833a5296dd3878ddb11b243f7f37a8fc
Parents: 2ea3b2d
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Wed Jun 20 19:07:03 2018 +0800
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Thu Jun 21 11:00:02 2018 +0530
----------------------------------------------------------------------
.../testsuite/dataload/TestLoadDataGeneral.scala | 16 ++++++++++++++++
.../store/CarbonFactDataHandlerColumnar.java | 7 ++++++-
2 files changed, 22 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/091a28bf/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
index 688928f..8b51090 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
@@ -259,6 +259,22 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterEach {
CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_DIRECT_WRITE_HDFS,
originStatus)
}
+
+ test("test data loading with page size less than 32000") {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.BLOCKLET_SIZE, "16000")
+
+ val testData = s"$resourcesPath/sample.csv"
+ sql(s"LOAD DATA LOCAL INPATH '$testData' into table loadtest")
+ checkAnswer(
+ sql("SELECT COUNT(*) FROM loadtest"),
+ Seq(Row(6))
+ )
+
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.BLOCKLET_SIZE,
+ CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL)
+ }
+
override def afterEach {
sql("DROP TABLE if exists loadtest")
sql("drop table if exists invalidMeasures")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/091a28bf/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index c0acadd..5fe3261 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -371,8 +371,13 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
this.pageSize = Integer.parseInt(CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.BLOCKLET_SIZE,
CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL));
+ // support less than 32000 rows in one page, because we support super long string,
+ // if it is long enough, a clomun page with 32000 rows will exceed 2GB
if (version == ColumnarFormatVersion.V3) {
- this.pageSize = CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ this.pageSize =
+ pageSize < CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT ?
+ pageSize :
+ CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
}
LOGGER.info("Number of rows per column blocklet " + pageSize);
dataRows = new ArrayList<>(this.pageSize);
[11/50] [abbrv] carbondata git commit: [CARBONDATA-2557]
[CARBONDATA-2472] [CARBONDATA-2570] Improve Carbon Reader performance on S3
and fixed datamap clear issue in reader
Posted by ja...@apache.org.
[CARBONDATA-2557] [CARBONDATA-2472] [CARBONDATA-2570] Improve Carbon Reader performance on S3 and fixed datamap clear issue in reader
[CARBONDATA-2557] [CARBONDATA-2472] Problem : CarbonReaderBuilder.build() is slower in s3. It takes around 8 seconds to finish build()
Solution: S3 is slow in listFiles, open, FileExist, getCarbonFile operations. So, List down all the calls of those API in the reader flow and remove the redundant checks.
[CARBONDATA-2570] Problem : Carbon SDK Reader, second time reader instance have an issue in cluster test
Solution: Blocklet datamap's of first time reader is not cleared properly in the cluster. Need to change the API to clear the blocklet datamap.
so change
DataMapStoreManager.getInstance().getDefaultDataMap(queryModel.getTable()).clear();
to
DataMapStoreManager.getInstance().clearDataMaps(queryModel.getTable().getAbsoluteTableIdentifie());
This closes #2345
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5f68a792
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5f68a792
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5f68a792
Branch: refs/heads/carbonstore
Commit: 5f68a792f2e83d15379740f715cf05d7ae9aaa05
Parents: 2f23486
Author: ajantha-bhat <aj...@gmail.com>
Authored: Sun May 27 22:49:23 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Jun 5 16:23:27 2018 +0530
----------------------------------------------------------------------
.../core/datamap/dev/CacheableDataMap.java | 6 +-
.../core/datastore/SegmentTaskIndexStore.java | 2 +-
.../indexstore/BlockletDataMapIndexStore.java | 84 +++++++------
.../TableBlockIndexUniqueIdentifierWrapper.java | 52 ++++++++
.../blockletindex/BlockletDataMapFactory.java | 122 ++++++++-----------
.../blockletindex/SegmentIndexFileStore.java | 15 +++
.../core/metadata/schema/table/CarbonTable.java | 60 ++++-----
.../LatestFilesReadCommittedScope.java | 19 +--
.../SegmentUpdateStatusManager.java | 15 ++-
.../core/util/BlockletDataMapUtil.java | 50 +++++++-
.../apache/carbondata/core/util/CarbonUtil.java | 30 +++++
.../TestBlockletDataMapFactory.java | 13 +-
docs/sdk-guide.md | 10 --
.../examples/sdk/CarbonReaderExample.java | 1 -
.../carbondata/hadoop/CarbonRecordReader.java | 3 +-
.../hadoop/api/CarbonFileInputFormat.java | 97 ++++-----------
.../hadoop/api/CarbonInputFormat.java | 24 ++++
...FileInputFormatWithExternalCarbonTable.scala | 2 +-
...tCreateTableUsingSparkCarbonFileFormat.scala | 2 +-
.../TestNonTransactionalCarbonTable.scala | 11 +-
.../sdk/file/CarbonReaderBuilder.java | 51 ++------
.../carbondata/sdk/file/CarbonReaderTest.java | 4 +-
22 files changed, 375 insertions(+), 298 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/main/java/org/apache/carbondata/core/datamap/dev/CacheableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/CacheableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/CacheableDataMap.java
index dba0840..e292c60 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/CacheableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/CacheableDataMap.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.indexstore.BlockletDataMapIndexWrapper;
-import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
+import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper;
import org.apache.carbondata.core.memory.MemoryException;
/**
@@ -33,10 +33,10 @@ public interface CacheableDataMap {
/**
* Add the blockletDataMapIndexWrapper to cache for key tableBlockIndexUniqueIdentifier
*
- * @param tableBlockIndexUniqueIdentifier
+ * @param tableBlockIndexUniqueIdentifierWrapper
* @param blockletDataMapIndexWrapper
*/
- void cache(TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier,
+ void cache(TableBlockIndexUniqueIdentifierWrapper tableBlockIndexUniqueIdentifierWrapper,
BlockletDataMapIndexWrapper blockletDataMapIndexWrapper) throws IOException, MemoryException;
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
index d325f21..c642091 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
@@ -91,7 +91,7 @@ public class SegmentTaskIndexStore
segmentTaskIndexWrapper =
loadAndGetTaskIdToSegmentsMap(
tableSegmentUniqueIdentifier.getSegmentToTableBlocksInfos(),
- CarbonTable.buildFromTablePath("name", "path", false),
+ CarbonTable.buildDummyTable("path"),
tableSegmentUniqueIdentifier);
} catch (IndexBuilderException e) {
throw new IOException(e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
index db49976..71a9b5a 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -41,7 +41,7 @@ import org.apache.carbondata.core.util.BlockletDataMapUtil;
* blocks
*/
public class BlockletDataMapIndexStore
- implements Cache<TableBlockIndexUniqueIdentifier, BlockletDataMapIndexWrapper> {
+ implements Cache<TableBlockIndexUniqueIdentifierWrapper, BlockletDataMapIndexWrapper> {
private static final LogService LOGGER =
LogServiceFactory.getLogService(BlockletDataMapIndexStore.class.getName());
/**
@@ -68,8 +68,10 @@ public class BlockletDataMapIndexStore
}
@Override
- public BlockletDataMapIndexWrapper get(TableBlockIndexUniqueIdentifier identifier)
+ public BlockletDataMapIndexWrapper get(TableBlockIndexUniqueIdentifierWrapper identifierWrapper)
throws IOException {
+ TableBlockIndexUniqueIdentifier identifier =
+ identifierWrapper.getTableBlockIndexUniqueIdentifier();
String lruCacheKey = identifier.getUniqueTableSegmentIdentifier();
BlockletDataMapIndexWrapper blockletDataMapIndexWrapper =
(BlockletDataMapIndexWrapper) lruCache.get(lruCacheKey);
@@ -84,7 +86,7 @@ public class BlockletDataMapIndexStore
// if the identifier is not a merge file we can directly load the datamaps
if (identifier.getMergeIndexFileName() == null) {
Map<String, BlockMetaInfo> blockMetaInfoMap = BlockletDataMapUtil
- .getBlockMetaInfoMap(identifier, indexFileStore, filesRead,
+ .getBlockMetaInfoMap(identifierWrapper, indexFileStore, filesRead,
carbonDataFileBlockMetaInfoMapping);
BlockletDataMap blockletDataMap =
loadAndGetDataMap(identifier, indexFileStore, blockMetaInfoMap);
@@ -96,9 +98,10 @@ public class BlockletDataMapIndexStore
BlockletDataMapUtil.getIndexFileIdentifiersFromMergeFile(identifier, indexFileStore);
for (TableBlockIndexUniqueIdentifier blockIndexUniqueIdentifier :
tableBlockIndexUniqueIdentifiers) {
- Map<String, BlockMetaInfo> blockMetaInfoMap = BlockletDataMapUtil
- .getBlockMetaInfoMap(blockIndexUniqueIdentifier, indexFileStore, filesRead,
- carbonDataFileBlockMetaInfoMapping);
+ Map<String, BlockMetaInfo> blockMetaInfoMap = BlockletDataMapUtil.getBlockMetaInfoMap(
+ new TableBlockIndexUniqueIdentifierWrapper(blockIndexUniqueIdentifier,
+ identifierWrapper.getCarbonTable()), indexFileStore, filesRead,
+ carbonDataFileBlockMetaInfoMapping);
BlockletDataMap blockletDataMap =
loadAndGetDataMap(blockIndexUniqueIdentifier, indexFileStore, blockMetaInfoMap);
dataMaps.add(blockletDataMap);
@@ -119,26 +122,28 @@ public class BlockletDataMapIndexStore
return blockletDataMapIndexWrapper;
}
- @Override
- public List<BlockletDataMapIndexWrapper> getAll(
- List<TableBlockIndexUniqueIdentifier> tableSegmentUniqueIdentifiers) throws IOException {
+ @Override public List<BlockletDataMapIndexWrapper> getAll(
+ List<TableBlockIndexUniqueIdentifierWrapper> tableSegmentUniqueIdentifiers)
+ throws IOException {
List<BlockletDataMapIndexWrapper> blockletDataMapIndexWrappers =
new ArrayList<>(tableSegmentUniqueIdentifiers.size());
- List<TableBlockIndexUniqueIdentifier> missedIdentifiers = new ArrayList<>();
+ List<TableBlockIndexUniqueIdentifierWrapper> missedIdentifiersWrapper = new ArrayList<>();
BlockletDataMapIndexWrapper blockletDataMapIndexWrapper = null;
// Get the datamaps for each indexfile from cache.
try {
- for (TableBlockIndexUniqueIdentifier identifier : tableSegmentUniqueIdentifiers) {
- BlockletDataMapIndexWrapper dataMapIndexWrapper = getIfPresent(identifier);
+ for (TableBlockIndexUniqueIdentifierWrapper
+ identifierWrapper : tableSegmentUniqueIdentifiers) {
+ BlockletDataMapIndexWrapper dataMapIndexWrapper =
+ getIfPresent(identifierWrapper);
if (dataMapIndexWrapper != null) {
blockletDataMapIndexWrappers.add(dataMapIndexWrapper);
} else {
- missedIdentifiers.add(identifier);
+ missedIdentifiersWrapper.add(identifierWrapper);
}
}
- if (missedIdentifiers.size() > 0) {
- for (TableBlockIndexUniqueIdentifier identifier : missedIdentifiers) {
- blockletDataMapIndexWrapper = get(identifier);
+ if (missedIdentifiersWrapper.size() > 0) {
+ for (TableBlockIndexUniqueIdentifierWrapper identifierWrapper : missedIdentifiersWrapper) {
+ blockletDataMapIndexWrapper = get(identifierWrapper);
blockletDataMapIndexWrappers.add(blockletDataMapIndexWrapper);
}
}
@@ -151,37 +156,40 @@ public class BlockletDataMapIndexStore
}
throw new IOException("Problem in loading segment blocks.", e);
}
+
return blockletDataMapIndexWrappers;
}
/**
* returns the SegmentTaskIndexWrapper
*
- * @param tableSegmentUniqueIdentifier
+ * @param tableSegmentUniqueIdentifierWrapper
* @return
*/
- @Override
- public BlockletDataMapIndexWrapper getIfPresent(
- TableBlockIndexUniqueIdentifier tableSegmentUniqueIdentifier) {
+ @Override public BlockletDataMapIndexWrapper getIfPresent(
+ TableBlockIndexUniqueIdentifierWrapper tableSegmentUniqueIdentifierWrapper) {
return (BlockletDataMapIndexWrapper) lruCache.get(
- tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier());
+ tableSegmentUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier()
+ .getUniqueTableSegmentIdentifier());
}
/**
* method invalidate the segment cache for segment
*
- * @param tableSegmentUniqueIdentifier
+ * @param tableSegmentUniqueIdentifierWrapper
*/
- @Override
- public void invalidate(TableBlockIndexUniqueIdentifier tableSegmentUniqueIdentifier) {
- lruCache.remove(tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier());
+ @Override public void invalidate(
+ TableBlockIndexUniqueIdentifierWrapper tableSegmentUniqueIdentifierWrapper) {
+ lruCache.remove(tableSegmentUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier()
+ .getUniqueTableSegmentIdentifier());
}
@Override
- public void put(TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier,
+ public void put(TableBlockIndexUniqueIdentifierWrapper tableBlockIndexUniqueIdentifierWrapper,
BlockletDataMapIndexWrapper wrapper) throws IOException, MemoryException {
String uniqueTableSegmentIdentifier =
- tableBlockIndexUniqueIdentifier.getUniqueTableSegmentIdentifier();
+ tableBlockIndexUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier()
+ .getUniqueTableSegmentIdentifier();
Object lock = segmentLockMap.get(uniqueTableSegmentIdentifier);
if (lock == null) {
lock = addAndGetSegmentLock(uniqueTableSegmentIdentifier);
@@ -190,16 +198,16 @@ public class BlockletDataMapIndexStore
// as in that case clearing unsafe memory need to be taken card. If at all datamap entry
// in the cache need to be overwritten then use the invalidate interface
// and then use the put interface
- if (null == getIfPresent(tableBlockIndexUniqueIdentifier)) {
+ if (null == getIfPresent(tableBlockIndexUniqueIdentifierWrapper)) {
synchronized (lock) {
- if (null == getIfPresent(tableBlockIndexUniqueIdentifier)) {
+ if (null == getIfPresent(tableBlockIndexUniqueIdentifierWrapper)) {
List<BlockletDataMap> dataMaps = wrapper.getDataMaps();
try {
for (BlockletDataMap blockletDataMap: dataMaps) {
blockletDataMap.convertToUnsafeDMStore();
}
- lruCache.put(tableBlockIndexUniqueIdentifier.getUniqueTableSegmentIdentifier(), wrapper,
- wrapper.getMemorySize());
+ lruCache.put(tableBlockIndexUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier()
+ .getUniqueTableSegmentIdentifier(), wrapper, wrapper.getMemorySize());
} catch (Throwable e) {
// clear all the memory acquired by data map in case of any failure
for (DataMap blockletDataMap : dataMaps) {
@@ -264,14 +272,14 @@ public class BlockletDataMapIndexStore
/**
* The method clears the access count of table segments
*
- * @param tableSegmentUniqueIdentifiers
+ * @param tableSegmentUniqueIdentifiersWrapper
*/
- @Override
- public void clearAccessCount(
- List<TableBlockIndexUniqueIdentifier> tableSegmentUniqueIdentifiers) {
- for (TableBlockIndexUniqueIdentifier identifier : tableSegmentUniqueIdentifiers) {
- BlockletDataMap cacheable =
- (BlockletDataMap) lruCache.get(identifier.getUniqueTableSegmentIdentifier());
+ @Override public void clearAccessCount(
+ List<TableBlockIndexUniqueIdentifierWrapper> tableSegmentUniqueIdentifiersWrapper) {
+ for (TableBlockIndexUniqueIdentifierWrapper
+ identifierWrapper : tableSegmentUniqueIdentifiersWrapper) {
+ BlockletDataMap cacheable = (BlockletDataMap) lruCache.get(
+ identifierWrapper.getTableBlockIndexUniqueIdentifier().getUniqueTableSegmentIdentifier());
cacheable.clear();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifierWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifierWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifierWrapper.java
new file mode 100644
index 0000000..3411397
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifierWrapper.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.indexstore;
+
+import java.io.Serializable;
+
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+
+/**
+ * Class holds reference to TableBlockIndexUniqueIdentifier and carbonTable related info
+ * This is just a wrapper passed between methods like a context, This object must never be cached.
+ *
+ */
+public class TableBlockIndexUniqueIdentifierWrapper implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ // holds the reference to tableBlockIndexUniqueIdentifier
+ private TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier;
+
+ // holds the reference to CarbonTable
+ private CarbonTable carbonTable;
+
+ public TableBlockIndexUniqueIdentifierWrapper(
+ TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier, CarbonTable carbonTable) {
+ this.tableBlockIndexUniqueIdentifier = tableBlockIndexUniqueIdentifier;
+ this.carbonTable = carbonTable;
+ }
+
+ public TableBlockIndexUniqueIdentifier getTableBlockIndexUniqueIdentifier() {
+ return tableBlockIndexUniqueIdentifier;
+ }
+
+ public CarbonTable getCarbonTable() {
+ return carbonTable;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index 318fc6e..c434e2e 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -44,16 +44,12 @@ import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
+import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.converter.SchemaConverter;
-import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
-import org.apache.carbondata.core.metadata.schema.table.TableInfo;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.BlockletDataMapUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.events.Event;
@@ -81,7 +77,7 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
// segmentId -> list of index file
private Map<String, Set<TableBlockIndexUniqueIdentifier>> segmentMap = new ConcurrentHashMap<>();
- private Cache<TableBlockIndexUniqueIdentifier, BlockletDataMapIndexWrapper> cache;
+ private Cache<TableBlockIndexUniqueIdentifierWrapper, BlockletDataMapIndexWrapper> cache;
public BlockletDataMapFactory(CarbonTable carbonTable, DataMapSchema dataMapSchema) {
super(carbonTable, dataMapSchema);
@@ -104,11 +100,15 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
List<CoarseGrainDataMap> dataMaps = new ArrayList<>();
Set<TableBlockIndexUniqueIdentifier> identifiers =
getTableBlockIndexUniqueIdentifiers(segment);
- List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
+ List<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers =
new ArrayList<>(identifiers.size());
- tableBlockIndexUniqueIdentifiers.addAll(identifiers);
+ for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : identifiers) {
+ tableBlockIndexUniqueIdentifierWrappers.add(
+ new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier,
+ this.getCarbonTable()));
+ }
List<BlockletDataMapIndexWrapper> blockletDataMapIndexWrappers =
- cache.getAll(tableBlockIndexUniqueIdentifiers);
+ cache.getAll(tableBlockIndexUniqueIdentifierWrappers);
for (BlockletDataMapIndexWrapper wrapper : blockletDataMapIndexWrappers) {
dataMaps.addAll(wrapper.getDataMaps());
}
@@ -120,12 +120,6 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
segmentMap.get(segment.getSegmentNo());
if (tableBlockIndexUniqueIdentifiers == null) {
- CarbonTable carbonTable = this.getCarbonTable();
- if (!carbonTable.getTableInfo().isTransactionalTable()) {
- // For NonTransactional table, compare the schema of all index files with inferred schema.
- // If there is a mismatch throw exception. As all files must be of same schema.
- validateSchemaForNewTranscationalTableFiles(segment, carbonTable);
- }
tableBlockIndexUniqueIdentifiers =
BlockletDataMapUtil.getTableBlockUniqueIdentifiers(segment);
segmentMap.put(segment.getSegmentNo(), tableBlockIndexUniqueIdentifiers);
@@ -133,46 +127,6 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
return tableBlockIndexUniqueIdentifiers;
}
- private void validateSchemaForNewTranscationalTableFiles(Segment segment, CarbonTable carbonTable)
- throws IOException {
- SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
- Map<String, String> indexFiles = segment.getCommittedIndexFile();
- for (Map.Entry<String, String> indexFileEntry : indexFiles.entrySet()) {
- Path indexFile = new Path(indexFileEntry.getKey());
- org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil.inferSchemaFromIndexFile(
- indexFile.toString(), carbonTable.getTableName());
- TableInfo wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(
- tableInfo, identifier.getDatabaseName(),
- identifier.getTableName(),
- identifier.getTablePath());
- List<ColumnSchema> indexFileColumnList =
- wrapperTableInfo.getFactTable().getListOfColumns();
- List<ColumnSchema> tableColumnList =
- carbonTable.getTableInfo().getFactTable().getListOfColumns();
- if (!isSameColumnSchemaList(indexFileColumnList, tableColumnList)) {
- LOG.error("Schema of " + indexFile.getName()
- + " doesn't match with the table's schema");
- throw new IOException("All the files doesn't have same schema. "
- + "Unsupported operation on nonTransactional table. Check logs.");
- }
- }
- }
-
- private boolean isSameColumnSchemaList(List<ColumnSchema> indexFileColumnList,
- List<ColumnSchema> tableColumnList) {
- if (indexFileColumnList.size() != tableColumnList.size()) {
- LOG.error("Index file's column size is " + indexFileColumnList.size()
- + " but table's column size is " + tableColumnList.size());
- return false;
- }
- for (int i = 0; i < tableColumnList.size(); i++) {
- if (!indexFileColumnList.get(i).equalsWithStrictCheck(tableColumnList.get(i))) {
- return false;
- }
- }
- return true;
- }
-
/**
* Get the blocklet detail information based on blockletid, blockid and segmentid. This method is
* exclusively for BlockletDataMapFactory as detail information is only available in this
@@ -191,9 +145,16 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
}
Set<TableBlockIndexUniqueIdentifier> identifiers =
getTableBlockIndexUniqueIdentifiers(segment);
+ Set<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers =
+ new HashSet<>(identifiers.size());
+ for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : identifiers) {
+ tableBlockIndexUniqueIdentifierWrappers.add(
+ new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier,
+ this.getCarbonTable()));
+ }
// Retrieve each blocklets detail information from blocklet datamap
for (Blocklet blocklet : blocklets) {
- detailedBlocklets.add(getExtendedBlocklet(identifiers, blocklet));
+ detailedBlocklets.add(getExtendedBlocklet(tableBlockIndexUniqueIdentifierWrappers, blocklet));
}
return detailedBlocklets;
}
@@ -204,14 +165,24 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
if (blocklet instanceof ExtendedBlocklet) {
return (ExtendedBlocklet) blocklet;
}
- Set<TableBlockIndexUniqueIdentifier> identifiers = getTableBlockIndexUniqueIdentifiers(segment);
- return getExtendedBlocklet(identifiers, blocklet);
+ Set<TableBlockIndexUniqueIdentifier> identifiers =
+ getTableBlockIndexUniqueIdentifiers(segment);
+
+ Set<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers =
+ new HashSet<>(identifiers.size());
+ for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : identifiers) {
+ tableBlockIndexUniqueIdentifierWrappers.add(
+ new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier,
+ this.getCarbonTable()));
+ }
+ return getExtendedBlocklet(tableBlockIndexUniqueIdentifierWrappers, blocklet);
}
- private ExtendedBlocklet getExtendedBlocklet(Set<TableBlockIndexUniqueIdentifier> identifiers,
- Blocklet blocklet) throws IOException {
- for (TableBlockIndexUniqueIdentifier identifier : identifiers) {
- BlockletDataMapIndexWrapper wrapper = cache.get(identifier);
+ private ExtendedBlocklet getExtendedBlocklet(
+ Set<TableBlockIndexUniqueIdentifierWrapper> identifiersWrapper, Blocklet blocklet)
+ throws IOException {
+ for (TableBlockIndexUniqueIdentifierWrapper identifierWrapper : identifiersWrapper) {
+ BlockletDataMapIndexWrapper wrapper = cache.get(identifierWrapper);
List<BlockletDataMap> dataMaps = wrapper.getDataMaps();
for (DataMap dataMap : dataMaps) {
if (((BlockletDataMap) dataMap).getIndexFileName().startsWith(blocklet.getFilePath())) {
@@ -265,12 +236,14 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
Set<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segment.getSegmentNo());
if (blockIndexes != null) {
for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
- BlockletDataMapIndexWrapper wrapper = cache.getIfPresent(blockIndex);
+ TableBlockIndexUniqueIdentifierWrapper blockIndexWrapper =
+ new TableBlockIndexUniqueIdentifierWrapper(blockIndex, this.getCarbonTable());
+ BlockletDataMapIndexWrapper wrapper = cache.getIfPresent(blockIndexWrapper);
if (null != wrapper) {
List<BlockletDataMap> dataMaps = wrapper.getDataMaps();
for (DataMap dataMap : dataMaps) {
if (dataMap != null) {
- cache.invalidate(blockIndex);
+ cache.invalidate(blockIndexWrapper);
dataMap.clear();
}
}
@@ -292,27 +265,28 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
public List<CoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
throws IOException {
BlockletDataMapDistributable mapDistributable = (BlockletDataMapDistributable) distributable;
- List<TableBlockIndexUniqueIdentifier> identifiers = new ArrayList<>();
+ List<TableBlockIndexUniqueIdentifierWrapper> identifiersWrapper = new ArrayList<>();
Path indexPath = new Path(mapDistributable.getFilePath());
String segmentNo = mapDistributable.getSegment().getSegmentNo();
if (indexPath.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
String parent = indexPath.getParent().toString();
- identifiers
- .add(new TableBlockIndexUniqueIdentifier(parent, indexPath.getName(), null, segmentNo));
+ identifiersWrapper.add(new TableBlockIndexUniqueIdentifierWrapper(
+ new TableBlockIndexUniqueIdentifier(parent, indexPath.getName(), null, segmentNo),
+ this.getCarbonTable()));
} else if (indexPath.getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
CarbonFile carbonFile = FileFactory.getCarbonFile(indexPath.toString());
String parentPath = carbonFile.getParentFile().getAbsolutePath();
List<String> indexFiles = fileStore.getIndexFilesFromMergeFile(carbonFile.getAbsolutePath());
for (String indexFile : indexFiles) {
- identifiers.add(
+ identifiersWrapper.add(new TableBlockIndexUniqueIdentifierWrapper(
new TableBlockIndexUniqueIdentifier(parentPath, indexFile, carbonFile.getName(),
- segmentNo));
+ segmentNo), this.getCarbonTable()));
}
}
List<CoarseGrainDataMap> dataMaps = new ArrayList<>();
try {
- List<BlockletDataMapIndexWrapper> wrappers = cache.getAll(identifiers);
+ List<BlockletDataMapIndexWrapper> wrappers = cache.getAll(identifiersWrapper);
for (BlockletDataMapIndexWrapper wrapper : wrappers) {
dataMaps.addAll(wrapper.getDataMaps());
}
@@ -356,9 +330,10 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
return false;
}
- @Override public void cache(TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier,
+ @Override
+ public void cache(TableBlockIndexUniqueIdentifierWrapper tableBlockIndexUniqueIdentifierWrapper,
BlockletDataMapIndexWrapper blockletDataMapIndexWrapper) throws IOException, MemoryException {
- cache.put(tableBlockIndexUniqueIdentifier, blockletDataMapIndexWrapper);
+ cache.put(tableBlockIndexUniqueIdentifierWrapper, blockletDataMapIndexWrapper);
}
@Override
@@ -373,7 +348,8 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
TableBlockIndexUniqueIdentifier validIdentifier = BlockletDataMapUtil
.filterIdentifiersBasedOnDistributable(tableBlockIndexUniqueIdentifiers,
(BlockletDataMapDistributable) distributable);
- if (null == cache.getIfPresent(validIdentifier)) {
+ if (null == cache.getIfPresent(
+ new TableBlockIndexUniqueIdentifierWrapper(validIdentifier, this.getCarbonTable()))) {
((BlockletDataMapDistributable) distributable)
.setTableBlockIndexUniqueIdentifier(validIdentifier);
distributablesToBeLoaded.add(distributable);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
index c2686d0..35e512d 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
@@ -323,6 +323,21 @@ public class SegmentIndexFileStore {
/**
* List all the index files of the segment.
*
+ * @param carbonFile directory
+ * @return
+ */
+ public static CarbonFile[] getCarbonIndexFiles(CarbonFile carbonFile) {
+ return carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return ((file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT) || file.getName()
+ .endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) && file.getSize() > 0);
+ }
+ });
+ }
+
+ /**
+ * List all the index files of the segment.
+ *
* @param segmentPath
* @return
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index ba051be..6949643 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -218,17 +218,9 @@ public class CarbonTable implements Serializable {
}
}
- public static CarbonTable buildFromTablePath(String tableName, String tablePath,
- boolean isTransactionalTable) throws IOException {
- if (isTransactionalTable) {
- return SchemaReader
- .readCarbonTableFromStore(AbsoluteTableIdentifier.from(tablePath, "default", tableName));
- } else {
- // Infer the schema from the Carbondata file.
- TableInfo tableInfoInfer =
- SchemaReader.inferSchema(AbsoluteTableIdentifier.from(tablePath, "null", "null"), false);
- return CarbonTable.buildFromTableInfo(tableInfoInfer);
- }
+ public static CarbonTable buildDummyTable(String tablePath) throws IOException {
+ TableInfo tableInfoInfer = CarbonUtil.buildDummyTableInfo(tablePath, "null", "null");
+ return CarbonTable.buildFromTableInfo(tableInfoInfer);
}
public static CarbonTable buildFromTablePath(String tableName, String dbName, String tablePath)
@@ -241,24 +233,7 @@ public class CarbonTable implements Serializable {
*/
public static CarbonTable buildFromTableInfo(TableInfo tableInfo) {
CarbonTable table = new CarbonTable();
- updateTableInfo(tableInfo);
- table.tableInfo = tableInfo;
- table.blockSize = tableInfo.getTableBlockSizeInMB();
- table.tableLastUpdatedTime = tableInfo.getLastUpdatedTime();
- table.tableUniqueName = tableInfo.getTableUniqueName();
- table.setTransactionalTable(tableInfo.isTransactionalTable());
- table.fillDimensionsAndMeasuresForTables(tableInfo.getFactTable());
- table.fillCreateOrderColumn(tableInfo.getFactTable().getTableName());
- if (tableInfo.getFactTable().getBucketingInfo() != null) {
- table.tableBucketMap.put(tableInfo.getFactTable().getTableName(),
- tableInfo.getFactTable().getBucketingInfo());
- }
- if (tableInfo.getFactTable().getPartitionInfo() != null) {
- table.tablePartitionMap.put(tableInfo.getFactTable().getTableName(),
- tableInfo.getFactTable().getPartitionInfo());
- }
- table.hasDataMapSchema =
- null != tableInfo.getDataMapSchemaList() && tableInfo.getDataMapSchemaList().size() > 0;
+ updateTableByTableInfo(table, tableInfo);
return table;
}
@@ -996,4 +971,31 @@ public class CarbonTable implements Serializable {
}
return indexColumn;
}
+
+ /**
+ * update the carbon table by using the passed tableInfo
+ *
+ * @param table
+ * @param tableInfo
+ */
+ public static void updateTableByTableInfo(CarbonTable table, TableInfo tableInfo) {
+ updateTableInfo(tableInfo);
+ table.tableInfo = tableInfo;
+ table.blockSize = tableInfo.getTableBlockSizeInMB();
+ table.tableLastUpdatedTime = tableInfo.getLastUpdatedTime();
+ table.tableUniqueName = tableInfo.getTableUniqueName();
+ table.setTransactionalTable(tableInfo.isTransactionalTable());
+ table.fillDimensionsAndMeasuresForTables(tableInfo.getFactTable());
+ table.fillCreateOrderColumn(tableInfo.getFactTable().getTableName());
+ if (tableInfo.getFactTable().getBucketingInfo() != null) {
+ table.tableBucketMap.put(tableInfo.getFactTable().getTableName(),
+ tableInfo.getFactTable().getBucketingInfo());
+ }
+ if (tableInfo.getFactTable().getPartitionInfo() != null) {
+ table.tablePartitionMap.put(tableInfo.getFactTable().getTableName(),
+ tableInfo.getFactTable().getPartitionInfo());
+ }
+ table.hasDataMapSchema =
+ null != tableInfo.getDataMapSchemaList() && tableInfo.getDataMapSchemaList().size() > 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java b/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java
index 6a1234e..63cfa21 100644
--- a/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java
+++ b/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java
@@ -23,7 +23,6 @@ import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
import org.apache.carbondata.core.mutate.UpdateVO;
@@ -157,28 +156,20 @@ public class LatestFilesReadCommittedScope implements ReadCommittedScope {
@Override public void takeCarbonIndexFileSnapShot() throws IOException {
// Read the current file Path get the list of indexes from the path.
CarbonFile file = FileFactory.getCarbonFile(carbonFilePath);
- CarbonFile[] files = file.listFiles(new CarbonFileFilter() {
- @Override
- public boolean accept(CarbonFile file) {
- return file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT) || file.getName()
- .endsWith(CarbonTablePath.CARBON_DATA_EXT) || file.getName().endsWith("Fact");
- }
- });
- if (files.length == 0) {
- // For nonTransactional table, files can be removed at any point of time.
- // So cannot assume files will be present
- throw new IOException("No files are present in the table location :" + carbonFilePath);
- }
Map<String, List<String>> indexFileStore = new HashMap<>();
Map<String, SegmentRefreshInfo> segmentTimestampUpdaterMap = new HashMap<>();
CarbonFile[] carbonIndexFiles = null;
if (file.isDirectory()) {
if (segmentId == null) {
- carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(carbonFilePath);
+ carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(file);
} else {
String segmentPath = CarbonTablePath.getSegmentPath(carbonFilePath, segmentId);
carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(segmentPath);
}
+ if (carbonIndexFiles.length == 0) {
+ throw new IOException(
+ "No Index files are present in the table location :" + carbonFilePath);
+ }
for (int i = 0; i < carbonIndexFiles.length; i++) {
// TODO. If Required to support merge index, then this code has to be modified.
// TODO. Nested File Paths.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index 1c53fbb..c2faadc 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -86,10 +86,19 @@ public class SegmentUpdateStatusManager {
this.identifier = table.getAbsoluteTableIdentifier();
// current it is used only for read function scenarios, as file update always requires to work
// on latest file status.
- segmentDetails = SegmentStatusManager.readLoadMetadata(
- CarbonTablePath.getMetadataPath(identifier.getTablePath()));
+ if (!table.getTableInfo().isTransactionalTable()) {
+ // fileExist is costly operation, so check based on table Type
+ segmentDetails = new LoadMetadataDetails[0];
+ } else {
+ segmentDetails = SegmentStatusManager.readLoadMetadata(
+ CarbonTablePath.getMetadataPath(identifier.getTablePath()));
+ }
isPartitionTable = table.isHivePartitionTable();
- updateDetails = readLoadMetadata();
+ if (segmentDetails.length != 0) {
+ updateDetails = readLoadMetadata();
+ } else {
+ updateDetails = new SegmentUpdateDetails[0];
+ }
populateMap();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java b/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java
index 0d28b9f..518cd03 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/BlockletDataMapUtil.java
@@ -33,20 +33,31 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.BlockMetaInfo;
import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
+import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable;
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
public class BlockletDataMapUtil {
+ private static final Log LOG = LogFactory.getLog(BlockletDataMapUtil.class);
+
public static Map<String, BlockMetaInfo> getBlockMetaInfoMap(
- TableBlockIndexUniqueIdentifier identifier, SegmentIndexFileStore indexFileStore,
- Set<String> filesRead, Map<String, BlockMetaInfo> fileNameToMetaInfoMapping)
- throws IOException {
+ TableBlockIndexUniqueIdentifierWrapper identifierWrapper,
+ SegmentIndexFileStore indexFileStore, Set<String> filesRead,
+ Map<String, BlockMetaInfo> fileNameToMetaInfoMapping) throws IOException {
+ boolean isTransactionalTable = true;
+ TableBlockIndexUniqueIdentifier identifier =
+ identifierWrapper.getTableBlockIndexUniqueIdentifier();
+ List<ColumnSchema> tableColumnList = null;
if (identifier.getMergeIndexFileName() != null
&& indexFileStore.getFileData(identifier.getIndexFileName()) == null) {
CarbonFile indexMergeFile = FileFactory.getCarbonFile(
@@ -67,7 +78,25 @@ public class BlockletDataMapUtil {
List<DataFileFooter> indexInfo = fileFooterConverter.getIndexInfo(
identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier
.getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()));
+ CarbonTable carbonTable = identifierWrapper.getCarbonTable();
+ if (carbonTable != null) {
+ isTransactionalTable = carbonTable.getTableInfo().isTransactionalTable();
+ tableColumnList =
+ carbonTable.getTableInfo().getFactTable().getListOfColumns();
+ }
for (DataFileFooter footer : indexInfo) {
+ if ((!isTransactionalTable) && (tableColumnList.size() != 0) &&
+ !isSameColumnSchemaList(footer.getColumnInTable(), tableColumnList)) {
+ LOG.error("Schema of " + identifier.getIndexFileName()
+ + " doesn't match with the table's schema");
+ throw new IOException("All the files doesn't have same schema. "
+ + "Unsupported operation on nonTransactional table. Check logs.");
+ }
+ if ((tableColumnList != null) && (tableColumnList.size() == 0)) {
+ // Carbon reader have used dummy columnSchema. Update it with inferred schema now
+ carbonTable.getTableInfo().getFactTable().setListOfColumns(footer.getColumnInTable());
+ CarbonTable.updateTableByTableInfo(carbonTable, carbonTable.getTableInfo());
+ }
String blockPath = footer.getBlockInfo().getTableBlockInfo().getFilePath();
if (null == blockMetaInfoMap.get(blockPath)) {
blockMetaInfoMap.put(blockPath, createBlockMetaInfo(fileNameToMetaInfoMapping, blockPath));
@@ -156,6 +185,7 @@ public class BlockletDataMapUtil {
* This method will the index files tableBlockIndexUniqueIdentifiers of a merge index file
*
* @param identifier
+ * @param segmentIndexFileStore
* @return
* @throws IOException
*/
@@ -177,4 +207,18 @@ public class BlockletDataMapUtil {
return tableBlockIndexUniqueIdentifiers;
}
+ private static boolean isSameColumnSchemaList(List<ColumnSchema> indexFileColumnList,
+ List<ColumnSchema> tableColumnList) {
+ if (indexFileColumnList.size() != tableColumnList.size()) {
+ LOG.error("Index file's column size is " + indexFileColumnList.size()
+ + " but table's column size is " + tableColumnList.size());
+ return false;
+ }
+ for (int i = 0; i < tableColumnList.size(); i++) {
+ if (!indexFileColumnList.get(i).equalsWithStrictCheck(tableColumnList.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 5a7bce3..e1e5e16 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -53,6 +53,7 @@ import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
+import org.apache.carbondata.core.metadata.converter.SchemaConverter;
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypeAdapter;
@@ -2371,6 +2372,35 @@ public final class CarbonUtil {
}
/**
+ * This method will prepare dummy tableInfo
+ *
+ * @param carbonDataFilePath
+ * @param tableName
+ * @return
+ */
+ public static TableInfo buildDummyTableInfo(String carbonDataFilePath,
+ String tableName, String dbName) {
+ // During SDK carbon Reader, This method will be called.
+ // This API will avoid IO operation to get the columnSchema list.
+ // ColumnSchema list will be filled during blocklet loading (where actual IO happens)
+ List<ColumnSchema> columnSchemaList = new ArrayList<>();
+ TableSchema tableSchema = getDummyTableSchema(tableName,columnSchemaList);
+ ThriftWrapperSchemaConverterImpl thriftWrapperSchemaConverter =
+ new ThriftWrapperSchemaConverterImpl();
+ org.apache.carbondata.format.TableSchema thriftFactTable =
+ thriftWrapperSchemaConverter.fromWrapperToExternalTableSchema(tableSchema);
+ org.apache.carbondata.format.TableInfo tableInfo =
+ new org.apache.carbondata.format.TableInfo(thriftFactTable,
+ new ArrayList<org.apache.carbondata.format.TableSchema>());
+ tableInfo.setDataMapSchemas(null);
+ SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
+ TableInfo wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(
+ tableInfo, dbName, tableName, carbonDataFilePath);
+ wrapperTableInfo.setTransactionalTable(false);
+ return wrapperTableInfo;
+ }
+
+ /**
* This method will infer the schema file from a given index file path
* @param indexFilePath
* @param tableName
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMapFactory.java b/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMapFactory.java
index dfbdd29..526f630 100644
--- a/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMapFactory.java
+++ b/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMapFactory.java
@@ -33,6 +33,7 @@ import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.indexstore.BlockletDataMapIndexWrapper;
import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
+import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -57,7 +58,9 @@ public class TestBlockletDataMapFactory {
private TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier;
- private Cache<TableBlockIndexUniqueIdentifier, BlockletDataMapIndexWrapper> cache;
+ private TableBlockIndexUniqueIdentifierWrapper tableBlockIndexUniqueIdentifierWrapper;
+
+ private Cache<TableBlockIndexUniqueIdentifierWrapper, BlockletDataMapIndexWrapper> cache;
@Before public void setUp()
throws ClassNotFoundException, IllegalAccessException, InvocationTargetException,
@@ -78,6 +81,8 @@ public class TestBlockletDataMapFactory {
tableBlockIndexUniqueIdentifier =
new TableBlockIndexUniqueIdentifier("/opt/store/default/carbon_table/Fact/Part0/Segment_0",
"0_batchno0-0-1521012756709.carbonindex", null, "0");
+ tableBlockIndexUniqueIdentifierWrapper =
+ new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, carbonTable);
cache = CacheProvider.getInstance().createCache(CacheType.DRIVER_BLOCKLET_DATAMAP);
}
@@ -86,12 +91,12 @@ public class TestBlockletDataMapFactory {
IllegalAccessException {
List<BlockletDataMap> dataMaps = new ArrayList<>();
Method method = BlockletDataMapFactory.class
- .getDeclaredMethod("cache", TableBlockIndexUniqueIdentifier.class,
+ .getDeclaredMethod("cache", TableBlockIndexUniqueIdentifierWrapper.class,
BlockletDataMapIndexWrapper.class);
method.setAccessible(true);
- method.invoke(blockletDataMapFactory, tableBlockIndexUniqueIdentifier,
+ method.invoke(blockletDataMapFactory, tableBlockIndexUniqueIdentifierWrapper,
new BlockletDataMapIndexWrapper(dataMaps));
- BlockletDataMapIndexWrapper result = cache.getIfPresent(tableBlockIndexUniqueIdentifier);
+ BlockletDataMapIndexWrapper result = cache.getIfPresent(tableBlockIndexUniqueIdentifierWrapper);
assert null != result;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/docs/sdk-guide.md
----------------------------------------------------------------------
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index 5dbb5ac..0f20dc3 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -460,16 +460,6 @@ Find example code at [CarbonReaderExample](https://github.com/apache/carbondata/
```
/**
- * Project all Columns for carbon reader
- *
- * @return CarbonReaderBuilder object
- * @throws IOException
- */
- public CarbonReaderBuilder projectAllColumns();
-```
-
-```
- /**
* Configure the transactional status of table
* If set to false, then reads the carbondata and carbonindex files from a flat folder structure.
* If set to true, then reads the carbondata and carbonindex files from segment folder structure.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
index 8d3ff0d..ada1a8c 100644
--- a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
+++ b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
@@ -116,7 +116,6 @@ public class CarbonReaderExample {
// Read data
CarbonReader reader2 = CarbonReader
.builder(path, "_temp")
- .projectAllColumns()
.build();
System.out.println("\nData:");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
index da84c00..4911e41 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
@@ -123,7 +123,8 @@ public class CarbonRecordReader<T> extends AbstractRecordReader<T> {
}
}
// Clear the datamap cache
- DataMapStoreManager.getInstance().getDefaultDataMap(queryModel.getTable()).clear();
+ DataMapStoreManager.getInstance()
+ .clearDataMaps(queryModel.getTable().getAbsoluteTableIdentifier());
// close read support
readSupport.close();
try {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java
index 8ed89d5..8755176 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java
@@ -23,26 +23,21 @@ import java.util.ArrayList;
import java.util.BitSet;
import java.util.LinkedList;
import java.util.List;
-import java.util.Map;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.PartitionInfo;
import org.apache.carbondata.core.metadata.schema.SchemaReader;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
-import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.readcommitter.LatestFilesReadCommittedScope;
import org.apache.carbondata.core.readcommitter.ReadCommittedScope;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
-import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.hadoop.CarbonInputSplit;
@@ -105,8 +100,10 @@ public class CarbonFileInputFormat<T> extends CarbonInputFormat<T> implements Se
*/
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
+
AbsoluteTableIdentifier identifier = getAbsoluteTableIdentifier(job.getConfiguration());
CarbonTable carbonTable = getOrCreateCarbonTable(job.getConfiguration());
+
if (null == carbonTable) {
throw new IOException("Missing/Corrupt schema file for table.");
}
@@ -115,6 +112,7 @@ public class CarbonFileInputFormat<T> extends CarbonInputFormat<T> implements Se
// get all valid segments and set them into the configuration
// check for externalTable segment (Segment_null)
// process and resolve the expression
+
ReadCommittedScope readCommittedScope = null;
if (carbonTable.isTransactionalTable()) {
readCommittedScope = new LatestFilesReadCommittedScope(
@@ -129,44 +127,33 @@ public class CarbonFileInputFormat<T> extends CarbonInputFormat<T> implements Se
FilterResolverIntf filterInterface = carbonTable.resolveFilter(filter);
- String segmentDir = null;
+ // if external table Segments are found, add it to the List
+ List<Segment> externalTableSegments = new ArrayList<Segment>();
+ Segment seg;
if (carbonTable.isTransactionalTable()) {
- segmentDir = CarbonTablePath.getSegmentPath(identifier.getTablePath(), "null");
+ // SDK some cases write into the Segment Path instead of Table Path i.e. inside
+ // the "Fact/Part0/Segment_null". The segment in this case is named as "null".
+ // The table is denoted by default as a transactional table and goes through
+ // the path of CarbonFileInputFormat. The above scenario is handled in the below code.
+ seg = new Segment("null", null, readCommittedScope);
+ externalTableSegments.add(seg);
} else {
- segmentDir = identifier.getTablePath();
- }
- FileFactory.FileType fileType = FileFactory.getFileType(segmentDir);
- if (FileFactory.isFileExist(segmentDir, fileType)) {
- // if external table Segments are found, add it to the List
- List<Segment> externalTableSegments = new ArrayList<Segment>();
- Segment seg;
- if (carbonTable.isTransactionalTable()) {
- // SDK some cases write into the Segment Path instead of Table Path i.e. inside
- // the "Fact/Part0/Segment_null". The segment in this case is named as "null".
- // The table is denoted by default as a transactional table and goes through
- // the path of CarbonFileInputFormat. The above scenario is handled in the below code.
- seg = new Segment("null", null, readCommittedScope);
+ LoadMetadataDetails[] loadMetadataDetails = readCommittedScope.getSegmentList();
+ for (LoadMetadataDetails load : loadMetadataDetails) {
+ seg = new Segment(load.getLoadName(), null, readCommittedScope);
externalTableSegments.add(seg);
- } else {
- LoadMetadataDetails[] loadMetadataDetails = readCommittedScope.getSegmentList();
- for (LoadMetadataDetails load : loadMetadataDetails) {
- seg = new Segment(load.getLoadName(), null, readCommittedScope);
- externalTableSegments.add(seg);
- }
}
-
- Map<String, String> indexFiles =
- new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir);
-
- if (indexFiles.size() == 0) {
- throw new RuntimeException("Index file not present to read the carbondata file");
- }
- // do block filtering and get split
- List<InputSplit> splits =
- getSplits(job, filterInterface, externalTableSegments, null, partitionInfo, null);
-
- return splits;
}
+ // do block filtering and get split
+ List<InputSplit> splits =
+ getSplits(job, filterInterface, externalTableSegments, null, partitionInfo, null);
+ if (getColumnProjection(job.getConfiguration()) == null) {
+ // If the user projection is empty, use default all columns as projections.
+ // All column name will be filled inside getSplits, so can update only here.
+ String[] projectionColumns = projectAllColumns(carbonTable);
+ setColumnProjection(job.getConfiguration(), projectionColumns);
+ }
+ return splits;
}
return null;
}
@@ -185,45 +172,13 @@ public class CarbonFileInputFormat<T> extends CarbonInputFormat<T> implements Se
numSegments = validSegments.size();
List<InputSplit> result = new LinkedList<InputSplit>();
- UpdateVO invalidBlockVOForSegmentId = null;
- Boolean isIUDTable = false;
-
- SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(carbonTable);
-
- isIUDTable = (updateStatusManager.getUpdateStatusDetails().length != 0);
// for each segment fetch blocks matching filter in Driver BTree
List<CarbonInputSplit> dataBlocksOfSegment =
getDataBlocksOfSegment(job, carbonTable, filterResolver, matchedPartitions,
validSegments, partitionInfo, oldPartitionIdList);
numBlocks = dataBlocksOfSegment.size();
- for (CarbonInputSplit inputSplit : dataBlocksOfSegment) {
-
- // Get the UpdateVO for those tables on which IUD operations being performed.
- if (isIUDTable) {
- invalidBlockVOForSegmentId =
- updateStatusManager.getInvalidTimestampRange(inputSplit.getSegmentId());
- }
- String[] deleteDeltaFilePath = null;
- if (isIUDTable) {
- // In case IUD is not performed in this table avoid searching for
- // invalidated blocks.
- if (CarbonUtil
- .isInvalidTableBlock(inputSplit.getSegmentId(), inputSplit.getPath().toString(),
- invalidBlockVOForSegmentId, updateStatusManager)) {
- continue;
- }
- // When iud is done then only get delete delta files for a block
- try {
- deleteDeltaFilePath = updateStatusManager
- .getDeleteDeltaFilePath(inputSplit.getPath().toString(), inputSplit.getSegmentId());
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
- inputSplit.setDeleteDeltaFiles(deleteDeltaFilePath);
- result.add(inputSplit);
- }
+ result.addAll(dataBlocksOfSegment);
return result;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
index 05c70f8..485b087 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
@@ -42,6 +42,7 @@ import org.apache.carbondata.core.metadata.schema.PartitionInfo;
import org.apache.carbondata.core.metadata.schema.partition.PartitionType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.profiler.ExplainCollector;
import org.apache.carbondata.core.scan.expression.Expression;
@@ -675,4 +676,27 @@ m filterExpression
return false;
}
}
+
+ /**
+ * Project all Columns for carbon reader
+ *
+ * @return String araay of columnNames
+ * @param carbonTable
+ */
+ public String[] projectAllColumns(CarbonTable carbonTable) {
+ List<ColumnSchema> colList = carbonTable.getTableInfo().getFactTable().getListOfColumns();
+ List<String> projectColumn = new ArrayList<>();
+ for (ColumnSchema cols : colList) {
+ if (cols.getSchemaOrdinal() != -1) {
+ projectColumn.add(cols.getColumnUniqueId());
+ }
+ }
+ String[] projectionColumns = new String[projectColumn.size()];
+ int i = 0;
+ for (String columnName : projectColumn) {
+ projectionColumns[i] = columnName;
+ i++;
+ }
+ return projectionColumns;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCarbonFileInputFormatWithExternalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCarbonFileInputFormatWithExternalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCarbonFileInputFormatWithExternalCarbonTable.scala
index e6d39d3..0e6f0c7 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCarbonFileInputFormatWithExternalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCarbonFileInputFormatWithExternalCarbonTable.scala
@@ -184,7 +184,7 @@ class TestCarbonFileInputFormatWithExternalCarbonTable extends QueryTest with Be
{
sql("select * from sdkOutputTable").show(false)
}
- assert(exception.getMessage().contains("Index file not present to read the carbondata file"))
+ assert(exception.getMessage().contains("Error while taking index snapshot"))
sql("DROP TABLE sdkOutputTable")
// drop table should not delete the files
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableUsingSparkCarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableUsingSparkCarbonFileFormat.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableUsingSparkCarbonFileFormat.scala
index 211bc8c..d7e500e 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableUsingSparkCarbonFileFormat.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableUsingSparkCarbonFileFormat.scala
@@ -346,7 +346,7 @@ class TestCreateTableUsingSparkCarbonFileFormat extends QueryTest with BeforeAnd
{
sql("select * from sdkOutputTable").show(false)
}
- assert(exception.getMessage().contains("Index file not present to read the carbondata file"))
+ assert(exception.getMessage().contains("Error while taking index snapshot"))
sql("DROP TABLE sdkOutputTable")
// drop table should not delete the files
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index 095d12d..14a63ca 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -993,7 +993,14 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("select * from sdkOutputTable").show(false)
}
assert(exception.getMessage()
- .contains("All the files doesn't have same schema"))
+ .contains("Problem in loading segment blocks."))
+
+ val exception1 =
+ intercept[IOException] {
+ sql("select count(*) from sdkOutputTable").show(false)
+ }
+ assert(exception1.getMessage()
+ .contains("Problem in loading segment blocks."))
sql("DROP TABLE sdkOutputTable")
// drop table should not delete the files
@@ -1025,7 +1032,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("select * from sdkOutputTable").show(false)
}
assert(exception.getMessage()
- .contains("All the files doesn't have same schema"))
+ .contains("Problem in loading segment blocks."))
sql("DROP TABLE sdkOutputTable")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
index 9d7470e..98aa6e0 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
@@ -26,7 +26,6 @@ import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.hadoop.api.CarbonFileInputFormat;
@@ -51,12 +50,6 @@ public class CarbonReaderBuilder {
private boolean isTransactionalTable;
/**
- * It will be true if use the projectAllColumns method，
- * it will be false if use the projection method
- */
- private boolean isProjectAllColumns = true;
-
- /**
* Construct a CarbonReaderBuilder with table path and table name
*
* @param tablePath table path
@@ -76,7 +69,6 @@ public class CarbonReaderBuilder {
public CarbonReaderBuilder projection(String[] projectionColumnNames) {
Objects.requireNonNull(projectionColumnNames);
this.projectionColumns = projectionColumnNames;
- isProjectAllColumns = false;
return this;
}
@@ -96,33 +88,6 @@ public class CarbonReaderBuilder {
}
/**
- * Project all Columns for carbon reader
- *
- * @return CarbonReaderBuilder object
- * @throws IOException
- */
- public CarbonReaderBuilder projectAllColumns() throws IOException {
- CarbonTable carbonTable = CarbonTable
- .buildFromTablePath(tableName, tablePath, isTransactionalTable);
-
- List<ColumnSchema> colList = carbonTable.getTableInfo().getFactTable().getListOfColumns();
- List<String> projectColumn = new ArrayList<String>();
- for (ColumnSchema cols : colList) {
- if (cols.getSchemaOrdinal() != -1) {
- projectColumn.add(cols.getColumnUniqueId());
- }
- }
- projectionColumns = new String[projectColumn.size()];
- int i = 0;
- for (String columnName : projectColumn) {
- projectionColumns[i] = columnName;
- i++;
- }
- isProjectAllColumns = true;
- return this;
- }
-
- /**
* Configure the filter expression for carbon reader
*
* @param filterExpression filter expression
@@ -209,8 +174,13 @@ public class CarbonReaderBuilder {
* @throws InterruptedException
*/
public <T> CarbonReader<T> build() throws IOException, InterruptedException {
- CarbonTable table = CarbonTable.buildFromTablePath(tableName, tablePath, isTransactionalTable);
-
+ // DB name is not applicable for SDK reader as, table will be never registered.
+ CarbonTable table;
+ if (isTransactionalTable) {
+ table = CarbonTable.buildFromTablePath(tableName, "default", tablePath);
+ } else {
+ table = CarbonTable.buildDummyTable(tablePath);
+ }
final CarbonFileInputFormat format = new CarbonFileInputFormat();
final Job job = new Job(new Configuration());
format.setTableInfo(job.getConfiguration(), table.getTableInfo());
@@ -220,10 +190,11 @@ public class CarbonReaderBuilder {
if (filterExpression != null) {
format.setFilterPredicates(job.getConfiguration(), filterExpression);
}
- if (isProjectAllColumns) {
- projectAllColumns();
+
+ if (projectionColumns != null) {
+ // set the user projection
+ format.setColumnProjection(job.getConfiguration(), projectionColumns);
}
- format.setColumnProjection(job.getConfiguration(), projectionColumns);
final List<InputSplit> splits =
format.getSplits(new JobContextImpl(job.getConfiguration(), new JobID()));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index db118cd..a8aa795 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -385,9 +385,8 @@ public class CarbonReaderTest extends TestCase {
// Write to a Non Transactional Table
TestUtil.writeFilesAndVerify(new Schema(fields), path, true, false);
- CarbonReader reader = CarbonReader.builder(path, "_temp").isTransactionalTable(true)
+ CarbonReader reader = CarbonReader.builder(path, "_temp")
.projection(new String[]{"name", "age"})
- .isTransactionalTable(false)
.build();
// expected output after sorting
@@ -892,7 +891,6 @@ public class CarbonReaderTest extends TestCase {
CarbonReader reader = CarbonReader
.builder(path, "_temp")
.isTransactionalTable(true)
- .projectAllColumns()
.build();
// expected output after sorting
[31/50] [abbrv] carbondata git commit: [CARBONDATA-2614] Fix the
error when using FG in search mode and the prune result is none
Posted by ja...@apache.org.
[CARBONDATA-2614] Fix the error when using FG in search mode and the prune result is none
the prune result is none, and can not set datamapWritePath, which will not generate bitSegGroup in org.apache.carbondata.core.scan.filter.executer.RowLevelFilterExecuterImpl#applyFilter(org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks, boolean), it mean the bitSetGroup is null. It will throw Fix the error when using FG in search mode and the prune result is none
This closes #2378
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5593d164
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5593d164
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5593d164
Branch: refs/heads/carbonstore
Commit: 5593d1646b075847a84ac2891fd25e78201451fe
Parents: 60dfdd3
Author: xubo245 <xu...@huawei.com>
Authored: Fri Jun 15 18:50:26 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Mon Jun 18 20:29:59 2018 +0800
----------------------------------------------------------------------
.../apache/carbondata/core/scan/model/QueryModel.java | 13 +++++++++++++
.../apache/carbondata/hadoop/CarbonRecordReader.java | 2 +-
.../testsuite/detailquery/SearchModeTestCase.scala | 1 +
.../carbondata/store/worker/SearchRequestHandler.java | 1 +
4 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
index de11d11..55dafb9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -109,6 +109,11 @@ public class QueryModel {
*/
private boolean requiredRowId;
+ /**
+ * whether it is FG with search mode
+ */
+ private boolean isFG;
+
private QueryModel(CarbonTable carbonTable) {
tableBlockInfos = new ArrayList<TableBlockInfo>();
invalidSegmentIds = new ArrayList<>();
@@ -370,6 +375,14 @@ public class QueryModel {
this.requiredRowId = requiredRowId;
}
+ public boolean isFG() {
+ return isFG;
+ }
+
+ public void setFG(boolean FG) {
+ isFG = FG;
+ }
+
@Override
public String toString() {
return String.format("scan on table %s.%s, %d projection columns with filter (%s)",
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
index 4911e41..6b56382 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
@@ -80,7 +80,7 @@ public class CarbonRecordReader<T> extends AbstractRecordReader<T> {
}
// It should use the exists tableBlockInfos if tableBlockInfos of queryModel is not empty
// otherwise the prune is no use before this method
- if (queryModel.getTableBlockInfos().isEmpty()) {
+ if (!queryModel.isFG()) {
List<TableBlockInfo> tableBlockInfoList = CarbonInputSplit.createBlocks(splitList);
queryModel.setTableBlockInfos(tableBlockInfoList);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
index 3e6adaf..001f6c0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
@@ -111,6 +111,7 @@ class SearchModeTestCase extends QueryTest with BeforeAndAfterAll {
}
test("test lucene datamap with search mode") {
+ sql("set carbon.search.enabled = true")
sql("DROP DATAMAP IF EXISTS dm ON TABLE main")
sql("CREATE DATAMAP dm ON TABLE main USING 'lucene' DMProperties('INDEX_COLUMNS'='id') ")
checkAnswer(sql("SELECT * FROM main WHERE TEXT_MATCH('id:100000')"),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
----------------------------------------------------------------------
diff --git a/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java b/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
index 4bfadce..0a3110e 100644
--- a/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
+++ b/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
@@ -207,6 +207,7 @@ public class SearchRequestHandler {
LOG.info(String.format("[SearchId:%d] pruned using FG DataMap, pruned blocks: %d", queryId,
blockToRead.size()));
queryModel.setTableBlockInfos(blockToRead);
+ queryModel.setFG(true);
return queryModel;
}
[40/50] [abbrv] carbondata git commit: [CARBONDATA-2420][32K] Support
string longer than 32000 characters
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
new file mode 100644
index 0000000..419b306
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
@@ -0,0 +1,279 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.carbondata.spark.testsuite.longstring
+
+import java.io.{File, PrintWriter}
+
+import org.apache.commons.lang3.RandomStringUtils
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class VarcharDataTypesBasicTestCase extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
+ private val longStringTable = "long_string_table"
+ private val inputDir = s"$resourcesPath${File.separator}varchartype${File.separator}"
+ private val fileName = s"longStringData.csv"
+ private val inputFile = s"$inputDir$fileName"
+ private val fileName_2g_column_page = s"longStringData_exceed_2gb_column_page.csv"
+ private val inputFile_2g_column_page = s"$inputDir$fileName_2g_column_page"
+ private val lineNum = 1000
+ private var content: Content = _
+ private var originMemorySize = CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
+ CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT)
+
+ case class Content(head: Int, desc_line_head: String, note_line_head: String,
+ mid: Int, desc_line_mid: String, note_line_mid: String,
+ tail: Int, desc_line_tail: String, note_line_tail: String)
+
+ override def beforeAll(): Unit = {
+ // for one 32000 lines * 32000 characters column page, it use about 1GB memory, but here we have only 1000 lines
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
+ CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT)
+ deleteFile(inputFile)
+ if (!new File(inputDir).exists()) {
+ new File(inputDir).mkdir()
+ }
+ content = createFile(inputFile, line = lineNum)
+ }
+
+ override def afterAll(): Unit = {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, originMemorySize)
+ deleteFile(inputFile)
+ deleteFile(inputFile_2g_column_page)
+ if (new File(inputDir).exists()) {
+ new File(inputDir).delete()
+ }
+ }
+
+ override def beforeEach(): Unit = {
+ sql(s"drop table if exists $longStringTable")
+ }
+
+ override def afterEach(): Unit = {
+ sql(s"drop table if exists $longStringTable")
+ }
+
+ private def prepareTable(): Unit = {
+ sql(
+ s"""
+ | CREATE TABLE if not exists $longStringTable(
+ | id INT, name STRING, description STRING, address STRING, note STRING
+ | ) STORED BY 'carbondata'
+ | TBLPROPERTIES('LONG_STRING_COLUMNS'='description, note', 'SORT_COLUMNS'='name')
+ |""".stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$inputFile' INTO TABLE $longStringTable
+ | OPTIONS('header'='false')
+ """.stripMargin)
+ }
+
+ private def checkQuery(): Unit = {
+ // query without long_string_column
+ checkAnswer(sql(s"SELECT id, name, address FROM $longStringTable where id = ${content.tail}"),
+ Row(content.tail, s"name_${content.tail}", s"address_${content.tail}"))
+ // query return long_string_column in the middle position
+ checkAnswer(sql(s"SELECT id, name, description, address FROM $longStringTable where id = ${content.head}"),
+ Row(content.head, s"name_${content.head}", content.desc_line_head, s"address_${content.head}"))
+ // query return long_string_column at last position
+ checkAnswer(sql(s"SELECT id, name, address, description FROM $longStringTable where id = ${content.mid}"),
+ Row(content.mid, s"name_${content.mid}", s"address_${content.mid}", content.desc_line_mid))
+ // query return 2 long_string_columns
+ checkAnswer(sql(s"SELECT id, name, note, address, description FROM $longStringTable where id = ${content.mid}"),
+ Row(content.mid, s"name_${content.mid}", content.note_line_mid, s"address_${content.mid}", content.desc_line_mid))
+ // query by simple string column
+ checkAnswer(sql(s"SELECT id, note, address, description FROM $longStringTable where name = 'name_${content.tail}'"),
+ Row(content.tail, content.note_line_tail, s"address_${content.tail}", content.desc_line_tail))
+ // query by long string column
+ checkAnswer(sql(s"SELECT id, name, address, description FROM $longStringTable where note = '${content.note_line_tail}'"),
+ Row(content.tail, s"name_${content.tail}", s"address_${content.tail}", content.desc_line_tail))
+ }
+
+ test("Load and query with long string datatype: safe sort & safe columnpage") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, "false")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "false")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE, "false")
+
+ prepareTable()
+ checkQuery()
+
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
+ CarbonCommonConstants.ENABLE_OFFHEAP_SORT_DEFAULT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE,
+ CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_DEFAULT)
+ }
+
+ test("Load and query with long string datatype: safe sort & unsafe column page") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, "false")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "false")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE, "true")
+
+ prepareTable()
+ checkQuery()
+
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
+ CarbonCommonConstants.ENABLE_OFFHEAP_SORT_DEFAULT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE,
+ CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_DEFAULT)
+ }
+
+ test("Load and query with long string datatype: unsafe sort & safe column page") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, "true")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE, "false")
+
+ prepareTable()
+ checkQuery()
+
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
+ CarbonCommonConstants.ENABLE_OFFHEAP_SORT_DEFAULT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE,
+ CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_DEFAULT)
+ }
+
+ test("Load and query with long string datatype: unsafe sort & unsafe column page") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, "true")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE, "true")
+
+ prepareTable()
+ checkQuery()
+
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
+ CarbonCommonConstants.ENABLE_OFFHEAP_SORT_DEFAULT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE,
+ CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_DEFAULT)
+ }
+
+ // ignore this test in CI, because it will need at least 4GB memory to run successfully
+ ignore("Exceed 2GB per column page for varchar datatype") {
+ deleteFile(inputFile_2g_column_page)
+ if (!new File(inputDir).exists()) {
+ new File(inputDir).mkdir()
+ }
+ // 7000000 characters with 3200 rows will exceed 2GB constraint for one column page.
+ content = createFile2(inputFile_2g_column_page, line = 3200, varcharLen = 700000)
+
+ sql(
+ s"""
+ | CREATE TABLE if not exists $longStringTable(
+ | id INT, name STRING, description STRING, address STRING
+ | ) STORED BY 'carbondata'
+ | TBLPROPERTIES('LONG_STRING_COLUMNS'='description', 'SORT_COLUMNS'='name')
+ |""".stripMargin)
+ val exceptionCaught = intercept[Exception] {
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$inputFile_2g_column_page' INTO TABLE $longStringTable
+ | OPTIONS('header'='false')
+ """.stripMargin)
+ }
+ // since after exception wrapper, we cannot get the root cause directly
+ }
+
+ // will create 2 long string columns
+ private def createFile(filePath: String, line: Int = 10000, start: Int = 0,
+ varcharLen: Int = Short.MaxValue + 1000): Content = {
+ val head = 0
+ val mid = line / 2
+ var tail = line - 1
+ var desc_line_head: String = ""
+ var desc_line_mid: String = ""
+ var desc_line_tail: String = ""
+ var note_line_head: String = ""
+ var note_line_mid: String = ""
+ var note_line_tail: String = ""
+ if (new File(filePath).exists()) {
+ deleteFile(filePath)
+ }
+ val write = new PrintWriter(new File(filePath))
+ for (i <- start until (start + line)) {
+ val description = RandomStringUtils.randomAlphabetic(varcharLen)
+ val note = RandomStringUtils.randomAlphabetic(varcharLen)
+ val line = s"$i,name_$i,$description,address_$i,$note"
+ if (head == i) {
+ desc_line_head = description
+ note_line_head = note
+ } else if (mid == i) {
+ desc_line_mid = description
+ note_line_mid = note
+ } else if (tail == i) {
+ desc_line_tail = description
+ note_line_tail = note
+ }
+ write.println(line)
+ }
+ write.close()
+ Content(head, desc_line_head, note_line_head,
+ mid, desc_line_mid, note_line_mid, tail,
+ desc_line_tail, note_line_tail)
+ }
+
+ // will only create 1 long string column
+ private def createFile2(filePath: String, line: Int = 10000, start: Int = 0,
+ varcharLen: Int = Short.MaxValue + 1000): Content = {
+ val head = 0
+ val mid = line / 2
+ var tail = line - 1
+ var desc_line_head: String = ""
+ var desc_line_mid: String = ""
+ var desc_line_tail: String = ""
+ if (new File(filePath).exists()) {
+ deleteFile(filePath)
+ }
+ val write = new PrintWriter(new File(filePath))
+ for (i <- start until (start + line)) {
+ val description = RandomStringUtils.randomAlphabetic(varcharLen)
+ val note = RandomStringUtils.randomAlphabetic(varcharLen)
+ val line = s"$i,name_$i,$description,address_$i"
+ if (head == i) {
+ desc_line_head = description
+ } else if (mid == i) {
+ desc_line_mid = description
+ } else if (tail == i) {
+ desc_line_tail = description
+ }
+ write.println(line)
+ }
+ write.close()
+ Content(head, desc_line_head, "",
+ mid, desc_line_mid, "", tail,
+ desc_line_tail, "")
+ }
+
+ private def deleteFile(filePath: String): Unit = {
+ val file = new File(filePath)
+ if (file.exists()) {
+ file.delete()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 1ccbf6a..6227655 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -115,6 +115,7 @@ object CarbonScalaUtil {
case CarbonDataTypes.BOOLEAN => BooleanType
case CarbonDataTypes.TIMESTAMP => TimestampType
case CarbonDataTypes.DATE => DateType
+ case CarbonDataTypes.VARCHAR => StringType
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
index 6673e18..6cd28c0 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
@@ -126,6 +126,7 @@ object DataTypeConverterUtil {
case "timestamp" => ThriftDataType.TIMESTAMP
case "array" => ThriftDataType.ARRAY
case "struct" => ThriftDataType.STRUCT
+ case "varchar" => ThriftDataType.VARCHAR
case _ => ThriftDataType.STRING
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 9af8817..0d53a73 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -280,7 +280,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
fields.zipWithIndex.foreach { case (field, index) =>
field.schemaOrdinal = index
}
- val (dims, msrs, noDictionaryDims, sortKeyDims) = extractDimAndMsrFields(
+ val (dims, msrs, noDictionaryDims, sortKeyDims, varcharColumns) = extractDimAndMsrFields(
fields, tableProperties)
// column properties
@@ -391,6 +391,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
reorderDimensions(dims.map(f => normalizeType(f)).map(f => addParent(f))),
msrs.map(f => normalizeType(f)),
Option(sortKeyDims),
+ Option(varcharColumns),
Option(noDictionaryDims),
Option(noInvertedIdxCols),
groupCols,
@@ -691,12 +692,31 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
* @return
*/
protected def extractDimAndMsrFields(fields: Seq[Field],
- tableProperties: Map[String, String]): (Seq[Field], Seq[Field], Seq[String], Seq[String]) = {
+ tableProperties: Map[String, String]):
+ (Seq[Field], Seq[Field], Seq[String], Seq[String], Seq[String]) = {
var dimFields: LinkedHashSet[Field] = LinkedHashSet[Field]()
var msrFields: Seq[Field] = Seq[Field]()
var dictExcludeCols: Array[String] = Array[String]()
var noDictionaryDims: Seq[String] = Seq[String]()
var dictIncludeCols: Seq[String] = Seq[String]()
+ var varcharCols: Seq[String] = Seq[String]()
+
+ // All long_string cols should be there in create table cols and should be of string data type
+ if (tableProperties.get(CarbonCommonConstants.LONG_STRING_COLUMNS).isDefined) {
+ varcharCols =
+ tableProperties(CarbonCommonConstants.LONG_STRING_COLUMNS).split(",").map(_.trim)
+ varcharCols.foreach { varcharCol =>
+ val exists = fields.exists(f => f.column.equalsIgnoreCase(varcharCol) &&
+ DataTypes.STRING.getName.equalsIgnoreCase(f.dataType.get))
+ if (!exists) {
+ throw new MalformedCarbonCommandException(
+ s"""
+ |${CarbonCommonConstants.LONG_STRING_COLUMNS}: $varcharCol does not exist in table
+ | or its data type is not string. Please check create table statement.
+ """.stripMargin)
+ }
+ }
+ }
// All columns in sortkey should be there in create table cols
val sortKeyOption = tableProperties.get(CarbonCommonConstants.SORT_COLUMNS)
@@ -727,6 +747,10 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
val errormsg = s"sort_columns is unsupported for $dataType datatype column: " + column
throw new MalformedCarbonCommandException(errormsg)
}
+ if (varcharCols.exists(x => x.equalsIgnoreCase(column))) {
+ throw new MalformedCarbonCommandException(
+ s"sort_columns is unsupported for long string datatype column $column")
+ }
}
}
@@ -824,9 +848,11 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
var sortKeyDims = sortKeyDimsTmp
if (sortKeyOption.isEmpty) {
- // if SORT_COLUMNS was not defined, add all dimension to SORT_COLUMNS.
+ // if SORT_COLUMNS was not defined,
+ // add all dimension(except long string columns) to SORT_COLUMNS.
dimFields.foreach { field =>
- if (!isComplexDimDictionaryExclude(field.dataType.get)) {
+ if (!isComplexDimDictionaryExclude(field.dataType.get) &&
+ !varcharCols.contains(field.column)) {
sortKeyDims :+= field.column
}
}
@@ -837,7 +863,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
} else {
tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, sortKeyDims.mkString(","))
}
- (dimFields.toSeq, msrFields, noDictionaryDims, sortKeyDims)
+ (dimFields.toSeq, msrFields, noDictionaryDims, sortKeyDims, varcharCols)
}
def isDefaultMeasure(dataType: Option[String]): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index d48db21..c77d0df 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -54,6 +54,7 @@ case class TableModel(
dimCols: Seq[Field],
msrCols: Seq[Field],
sortKeyDims: Option[Seq[String]],
+ varcharCols: Option[Seq[String]],
highcardinalitydims: Option[Seq[String]],
noInvertedIdxCols: Option[Seq[String]],
columnGroups: Seq[String],
@@ -212,9 +213,9 @@ class AlterTableColumnSchemaGenerator(
tableIdentifier: AbsoluteTableIdentifier,
sc: SparkContext) {
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
+ private val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
- def isSortColumn(columnName: String): Boolean = {
+ private def isSortColumn(columnName: String): Boolean = {
val sortColumns = alterTableModel.tableProperties.get("sort_columns")
if(sortColumns.isDefined) {
sortColumns.get.contains(columnName)
@@ -222,6 +223,16 @@ class AlterTableColumnSchemaGenerator(
true
}
}
+
+ private def isVarcharColumn(columnName: String): Boolean = {
+ val varcharColumns = alterTableModel.tableProperties.get("long_string_columns")
+ if (varcharColumns.isDefined) {
+ varcharColumns.get.contains(columnName)
+ } else {
+ false
+ }
+ }
+
def process: Seq[ColumnSchema] = {
val tableSchema = tableInfo.getFactTable
val tableCols = tableSchema.getListOfColumns.asScala
@@ -241,7 +252,8 @@ class AlterTableColumnSchemaGenerator(
field.schemaOrdinal + existingColsSize,
alterTableModel.highCardinalityDims,
alterTableModel.databaseName.getOrElse(dbName),
- isSortColumn(field.name.getOrElse(field.column)))
+ isSortColumn(field.name.getOrElse(field.column)),
+ isVarcharColumn(field.name.getOrElse(field.column)))
allColumns ++= Seq(columnSchema)
newCols ++= Seq(columnSchema)
})
@@ -351,14 +363,19 @@ object TableNewProcessor {
schemaOrdinal: Int,
highCardinalityDims: Seq[String],
databaseName: String,
- isSortColumn: Boolean = false): ColumnSchema = {
+ isSortColumn: Boolean = false,
+ isVarcharColumn: Boolean = false): ColumnSchema = {
val dataType = DataTypeConverterUtil.convertToCarbonType(field.dataType.getOrElse(""))
if (DataTypes.isDecimal(dataType)) {
dataType.asInstanceOf[DecimalType].setPrecision(field.precision)
dataType.asInstanceOf[DecimalType].setScale(field.scale)
}
val columnSchema = new ColumnSchema()
- columnSchema.setDataType(dataType)
+ if (isVarcharColumn) {
+ columnSchema.setDataType(DataTypes.VARCHAR)
+ } else {
+ columnSchema.setDataType(dataType)
+ }
val colName = field.name.getOrElse(field.column)
columnSchema.setColumnName(colName)
if (highCardinalityDims.contains(colName)) {
@@ -415,6 +432,11 @@ class TableNewProcessor(cm: TableModel) {
allColumns
}
+ // varchar column is a string column that in long_string_columns
+ private def isVarcharColumn(colName : String): Boolean = {
+ cm.varcharCols.get.contains(colName)
+ }
+
def getColumnSchema(
dataType: DataType,
colName: String,
@@ -450,6 +472,9 @@ class TableNewProcessor(cm: TableModel) {
columnSchema.setScale(field.scale)
columnSchema.setSchemaOrdinal(field.schemaOrdinal)
columnSchema.setSortColumn(false)
+ if (isVarcharColumn(colName)) {
+ columnSchema.setDataType(DataTypes.VARCHAR)
+ }
if(isParentColumnRelation) {
val dataMapField = map.get.get(field).get
columnSchema.setFunction(dataMapField.aggregateFunction)
@@ -517,7 +542,7 @@ class TableNewProcessor(cm: TableModel) {
val dictionaryIncludeCols = cm.tableProperties
.getOrElse(CarbonCommonConstants.DICTIONARY_INCLUDE, "")
- cm.dimCols.foreach { field =>
+ def addDimensionCol(field: Field): Unit = {
val sortField = cm.sortKeyDims.get.find(field.column equals _)
if (sortField.isEmpty) {
val encoders = if (getEncoderFromParent(field)) {
@@ -549,6 +574,12 @@ class TableNewProcessor(cm: TableModel) {
}
}
}
+ // dimensions that are not varchar
+ cm.dimCols.filter(field => !cm.varcharCols.get.contains(field.column))
+ .foreach(addDimensionCol(_))
+ // dimensions that are varchar
+ cm.dimCols.filter(field => cm.varcharCols.get.contains(field.column))
+ .foreach(addDimensionCol(_))
// check whether the column is a local dictionary column and set in column schema
if (null != cm.tableProperties) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
index 5739d3e..2f2048d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
@@ -201,6 +201,7 @@ case class CarbonRelation(
object CarbonMetastoreTypes extends RegexParsers {
protected lazy val primitiveType: Parser[DataType] =
"string" ^^^ StringType |
+ "varchar" ^^^ StringType |
"float" ^^^ FloatType |
"int" ^^^ IntegerType |
"tinyint" ^^^ ShortType |
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
index 9cf7fe4..3018e49 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
@@ -73,8 +73,10 @@ public class NonDictionaryFieldConverterImpl implements FieldConverter {
.getBytesBasedOnDataTypeForNoDictionaryColumn(dimensionValue, dataType, dateFormat);
if (dataType == DataTypes.STRING
&& value.length > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
- throw new CarbonDataLoadingException("Dataload failed, String size cannot exceed "
- + CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT + " bytes");
+ throw new CarbonDataLoadingException(String.format(
+ "Dataload failed, String size cannot exceed %d bytes,"
+ + " please consider long string data type",
+ CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT));
}
row.update(value, index);
} else {
@@ -82,8 +84,10 @@ public class NonDictionaryFieldConverterImpl implements FieldConverter {
.getDataDataTypeForNoDictionaryColumn(dimensionValue, dataType, dateFormat);
if (dataType == DataTypes.STRING
&& value.toString().length() > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
- throw new CarbonDataLoadingException("Dataload failed, String size cannot exceed "
- + CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT + " bytes");
+ throw new CarbonDataLoadingException(String.format(
+ "Dataload failed, String size cannot exceed %d bytes,"
+ + " please consider long string data type",
+ CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT));
}
row.update(value, index);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java b/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java
index 2e3479c..86c71a6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java
@@ -205,7 +205,9 @@ public class CSVInputFormat extends FileInputFormat<NullWritable, StringArrayWri
parserSettings.setSkipEmptyLines(
Boolean.valueOf(job.get(SKIP_EMPTY_LINE,
CarbonCommonConstants.CARBON_SKIP_EMPTY_LINE_DEFAULT)));
- parserSettings.setMaxCharsPerColumn(CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT);
+ // todo: will verify whether there is a performance degrade using -1 here
+ // parserSettings.setMaxCharsPerColumn(CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT);
+ parserSettings.setMaxCharsPerColumn(CarbonCommonConstants.MAX_CHARS_PER_COLUMN_INFINITY);
String maxColumns = job.get(MAX_COLUMNS, "" + DEFAULT_MAX_NUMBER_OF_COLUMNS_FOR_PARSING);
parserSettings.setMaxColumns(Integer.parseInt(maxColumns));
parserSettings.getFormat().setQuote(job.get(QUOTE, QUOTE_DEFAULT).charAt(0));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java b/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java
index 8d351cf..8bec099 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java
@@ -54,12 +54,13 @@ public class IntermediateSortTempRow {
/**
* deserialize from bytes array to get the no sort fields
* @param outDictNoSort stores the dict & no-sort fields
- * @param outNoDictNoSort stores the no-dict & no-sort fields, including complex
+ * @param outNoDictNoSortAndVarcharDims stores the no-dict & no-sort fields,
+ * including complex and varchar fields
* @param outMeasures stores the measure fields
* @param dataTypes data type for the measure
*/
- public void unpackNoSortFromBytes(int[] outDictNoSort, byte[][] outNoDictNoSort,
- Object[] outMeasures, DataType[] dataTypes) {
+ public void unpackNoSortFromBytes(int[] outDictNoSort, byte[][] outNoDictNoSortAndVarcharDims,
+ Object[] outMeasures, DataType[] dataTypes, int varcharDimCnt) {
ByteBuffer rowBuffer = ByteBuffer.wrap(noSortDimsAndMeasures);
// read dict_no_sort
int dictNoSortCnt = outDictNoSort.length;
@@ -68,12 +69,20 @@ public class IntermediateSortTempRow {
}
// read no_dict_no_sort (including complex)
- int noDictNoSortCnt = outNoDictNoSort.length;
+ int noDictNoSortCnt = outNoDictNoSortAndVarcharDims.length - varcharDimCnt;
for (int i = 0; i < noDictNoSortCnt; i++) {
short len = rowBuffer.getShort();
byte[] bytes = new byte[len];
rowBuffer.get(bytes);
- outNoDictNoSort[i] = bytes;
+ outNoDictNoSortAndVarcharDims[i] = bytes;
+ }
+
+ // read varchar dims
+ for (int i = 0; i < varcharDimCnt; i++) {
+ int len = rowBuffer.getInt();
+ byte[] bytes = new byte[len];
+ rowBuffer.get(bytes);
+ outNoDictNoSortAndVarcharDims[i + noDictNoSortCnt] = bytes;
}
// read measure
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java
index f31a2b9..bcf8a39 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java
@@ -46,6 +46,7 @@ public class SortStepRowHandler implements Serializable {
private int dictNoSortDimCnt = 0;
private int noDictSortDimCnt = 0;
private int noDictNoSortDimCnt = 0;
+ private int varcharDimCnt = 0;
private int measureCnt;
// indices for dict & sort dimension columns
@@ -56,6 +57,7 @@ public class SortStepRowHandler implements Serializable {
private int[] noDictSortDimIdx;
// indices for no-dict & no-sort dimension columns, including complex columns
private int[] noDictNoSortDimIdx;
+ private int[] varcharDimIdx;
// indices for measure columns
private int[] measureIdx;
@@ -70,11 +72,13 @@ public class SortStepRowHandler implements Serializable {
this.dictNoSortDimCnt = tableFieldStat.getDictNoSortDimCnt();
this.noDictSortDimCnt = tableFieldStat.getNoDictSortDimCnt();
this.noDictNoSortDimCnt = tableFieldStat.getNoDictNoSortDimCnt();
+ this.varcharDimCnt = tableFieldStat.getVarcharDimCnt();
this.measureCnt = tableFieldStat.getMeasureCnt();
this.dictSortDimIdx = tableFieldStat.getDictSortDimIdx();
this.dictNoSortDimIdx = tableFieldStat.getDictNoSortDimIdx();
this.noDictSortDimIdx = tableFieldStat.getNoDictSortDimIdx();
this.noDictNoSortDimIdx = tableFieldStat.getNoDictNoSortDimIdx();
+ this.varcharDimIdx = tableFieldStat.getVarcharDimIdx();
this.measureIdx = tableFieldStat.getMeasureIdx();
this.dataTypes = tableFieldStat.getMeasureDataType();
}
@@ -122,6 +126,10 @@ public class SortStepRowHandler implements Serializable {
for (int idx = 0; idx < this.noDictNoSortDimCnt; idx++) {
nonDictArray[idxAcc++] = (byte[]) row[this.noDictNoSortDimIdx[idx]];
}
+ // convert varchar dims
+ for (int idx = 0; idx < this.varcharDimCnt; idx++) {
+ nonDictArray[idxAcc++] = (byte[]) row[this.varcharDimIdx[idx]];
+ }
// convert measure data
for (int idx = 0; idx < this.measureCnt; idx++) {
@@ -146,13 +154,15 @@ public class SortStepRowHandler implements Serializable {
int[] dictDims
= new int[this.dictSortDimCnt + this.dictNoSortDimCnt];
byte[][] noDictArray
- = new byte[this.noDictSortDimCnt + this.noDictNoSortDimCnt][];
+ = new byte[this.noDictSortDimCnt + this.noDictNoSortDimCnt + this.varcharDimCnt][];
int[] dictNoSortDims = new int[this.dictNoSortDimCnt];
- byte[][] noDictNoSortDims = new byte[this.noDictNoSortDimCnt][];
+ byte[][] noDictNoSortAndVarcharDims
+ = new byte[this.noDictNoSortDimCnt + this.varcharDimCnt][];
Object[] measures = new Object[this.measureCnt];
- sortTempRow.unpackNoSortFromBytes(dictNoSortDims, noDictNoSortDims, measures, this.dataTypes);
+ sortTempRow.unpackNoSortFromBytes(dictNoSortDims, noDictNoSortAndVarcharDims, measures,
+ this.dataTypes, this.varcharDimCnt);
// dict dims
System.arraycopy(sortTempRow.getDictSortDims(), 0 , dictDims,
@@ -163,8 +173,8 @@ public class SortStepRowHandler implements Serializable {
// no dict dims, including complex
System.arraycopy(sortTempRow.getNoDictSortDims(), 0,
noDictArray, 0, this.noDictSortDimCnt);
- System.arraycopy(noDictNoSortDims, 0, noDictArray,
- this.noDictSortDimCnt, this.noDictNoSortDimCnt);
+ System.arraycopy(noDictNoSortAndVarcharDims, 0, noDictArray,
+ this.noDictSortDimCnt, this.noDictNoSortDimCnt + this.varcharDimCnt);
// measures are already here
@@ -428,6 +438,12 @@ public class SortStepRowHandler implements Serializable {
rowBuffer.putShort((short) bytes.length);
rowBuffer.put(bytes);
}
+ // convert varchar dims
+ for (int idx = 0; idx < this.varcharDimCnt; idx++) {
+ byte[] bytes = (byte[]) row[this.varcharDimIdx[idx]];
+ rowBuffer.putInt(bytes.length);
+ rowBuffer.put(bytes);
+ }
// convert measure
Object tmpValue;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index dde18a9..9dab181 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -92,6 +92,10 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
*/
private boolean[] noDictionaryColMapping;
/**
+ * boolean mapping for long string dimension
+ */
+ private boolean[] isVarcharDimMapping;
+ /**
* agg type defined for measures
*/
private DataType[] dataTypes;
@@ -353,13 +357,18 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
measureCount = carbonTable.getMeasureByTableName(tableName).size();
List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(tableName);
noDictionaryColMapping = new boolean[dimensions.size()];
+ isVarcharDimMapping = new boolean[dimensions.size()];
int i = 0;
+ int j = 0;
for (CarbonDimension dimension : dimensions) {
if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
i++;
continue;
}
noDictionaryColMapping[i++] = true;
+ if (dimension.getColumnSchema().getDataType() == DataTypes.VARCHAR) {
+ isVarcharDimMapping[j++] = true;
+ }
noDictionaryCount++;
}
dimensionColumnCount = dimensions.size();
@@ -387,7 +396,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
.createSortParameters(carbonTable, carbonLoadModel.getDatabaseName(), tableName,
dimensionColumnCount, segmentProperties.getComplexDimensions().size(), measureCount,
noDictionaryCount, segmentId,
- carbonLoadModel.getTaskNo(), noDictionaryColMapping, true);
+ carbonLoadModel.getTaskNo(), noDictionaryColMapping, isVarcharDimMapping, true);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
index 705350c..502fa05 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
@@ -111,7 +111,11 @@ public class SortParameters implements Serializable {
private boolean[] noDictionaryDimnesionColumn;
private boolean[] noDictionarySortColumn;
-
+ /**
+ * whether dimension is varchar data type.
+ * since all dimensions are string, we use an array of boolean instead of datatypes
+ */
+ private boolean[] isVarcharDimensionColumn;
private int numberOfSortColumns;
private int numberOfNoDictSortColumns;
@@ -143,6 +147,7 @@ public class SortParameters implements Serializable {
parameters.segmentId = segmentId;
parameters.taskNo = taskNo;
parameters.noDictionaryDimnesionColumn = noDictionaryDimnesionColumn;
+ parameters.isVarcharDimensionColumn = isVarcharDimensionColumn;
parameters.noDictionarySortColumn = noDictionarySortColumn;
parameters.numberOfSortColumns = numberOfSortColumns;
parameters.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
@@ -312,6 +317,14 @@ public class SortParameters implements Serializable {
this.noDictionaryDimnesionColumn = noDictionaryDimnesionColumn;
}
+ public boolean[] getIsVarcharDimensionColumn() {
+ return isVarcharDimensionColumn;
+ }
+
+ public void setIsVarcharDimensionColumn(boolean[] isVarcharDimensionColumn) {
+ this.isVarcharDimensionColumn = isVarcharDimensionColumn;
+ }
+
public int getNumberOfCores() {
return numberOfCores;
}
@@ -371,6 +384,8 @@ public class SortParameters implements Serializable {
.getComplexNonDictionaryColumnCount());
parameters.setNoDictionaryDimnesionColumn(
CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields()));
+ parameters.setIsVarcharDimensionColumn(
+ CarbonDataProcessorUtil.getIsVarcharColumnMapping(configuration.getDataFields()));
parameters.setBatchSortSizeinMb(CarbonDataProcessorUtil.getBatchSortSizeinMb(configuration));
parameters.setNumberOfSortColumns(configuration.getNumberOfSortColumns());
@@ -461,7 +476,8 @@ public class SortParameters implements Serializable {
public static SortParameters createSortParameters(CarbonTable carbonTable, String databaseName,
String tableName, int dimColCount, int complexDimColCount, int measureColCount,
int noDictionaryCount, String segmentId, String taskNo,
- boolean[] noDictionaryColMaping, boolean isCompactionFlow) {
+ boolean[] noDictionaryColMaping, boolean[] isVarcharDimensionColumn,
+ boolean isCompactionFlow) {
SortParameters parameters = new SortParameters();
CarbonProperties carbonProperties = CarbonProperties.getInstance();
parameters.setDatabaseName(databaseName);
@@ -476,6 +492,7 @@ public class SortParameters implements Serializable {
parameters.setNumberOfNoDictSortColumns(carbonTable.getNumberOfNoDictSortColumns());
parameters.setComplexDimColCount(complexDimColCount);
parameters.setNoDictionaryDimnesionColumn(noDictionaryColMaping);
+ parameters.setIsVarcharDimensionColumn(isVarcharDimensionColumn);
parameters.setObserver(new SortObserver());
// get sort buffer size
parameters.setSortBufferSize(Integer.parseInt(carbonProperties
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
index 0d1303a..094bd83 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
@@ -33,9 +33,13 @@ public class TableFieldStat implements Serializable {
private int dictSortDimCnt = 0;
private int dictNoSortDimCnt = 0;
private int noDictSortDimCnt = 0;
+ // for columns that are no_dict_dim and no_sort_dim and complex, except the varchar dims
private int noDictNoSortDimCnt = 0;
+ // for columns that are varchar data type
+ private int varcharDimCnt = 0;
// whether sort column is of dictionary type or not
private boolean[] isSortColNoDictFlags;
+ private boolean[] isVarcharDimFlags;
private int measureCnt;
private DataType[] measureDataType;
@@ -47,6 +51,8 @@ public class TableFieldStat implements Serializable {
private int[] noDictSortDimIdx;
// indices for no-dict & no-sort dimension columns, including complex columns
private int[] noDictNoSortDimIdx;
+ // indices for varchar dimension columns
+ private int[] varcharDimIdx;
// indices for measure columns
private int[] measureIdx;
@@ -55,6 +61,7 @@ public class TableFieldStat implements Serializable {
int complexDimCnt = sortParameters.getComplexDimColCount();
int dictDimCnt = sortParameters.getDimColCount() - noDictDimCnt;
this.isSortColNoDictFlags = sortParameters.getNoDictionarySortColumn();
+ this.isVarcharDimFlags = sortParameters.getIsVarcharDimensionColumn();
int sortColCnt = isSortColNoDictFlags.length;
for (boolean flag : isSortColNoDictFlags) {
if (flag) {
@@ -66,22 +73,33 @@ public class TableFieldStat implements Serializable {
this.measureCnt = sortParameters.getMeasureColCount();
this.measureDataType = sortParameters.getMeasureDataType();
+ for (boolean flag : isVarcharDimFlags) {
+ if (flag) {
+ varcharDimCnt++;
+ }
+ }
+
// be careful that the default value is 0
this.dictSortDimIdx = new int[dictSortDimCnt];
this.dictNoSortDimIdx = new int[dictDimCnt - dictSortDimCnt];
this.noDictSortDimIdx = new int[noDictSortDimCnt];
- this.noDictNoSortDimIdx = new int[noDictDimCnt + complexDimCnt - noDictSortDimCnt];
+ this.noDictNoSortDimIdx = new int[noDictDimCnt + complexDimCnt - noDictSortDimCnt
+ - varcharDimCnt];
+ this.varcharDimIdx = new int[varcharDimCnt];
this.measureIdx = new int[measureCnt];
int tmpNoDictSortCnt = 0;
int tmpNoDictNoSortCnt = 0;
int tmpDictSortCnt = 0;
int tmpDictNoSortCnt = 0;
+ int tmpVarcharCnt = 0;
boolean[] isDimNoDictFlags = sortParameters.getNoDictionaryDimnesionColumn();
for (int i = 0; i < isDimNoDictFlags.length; i++) {
if (isDimNoDictFlags[i]) {
- if (i < sortColCnt && isSortColNoDictFlags[i]) {
+ if (isVarcharDimFlags[i]) {
+ varcharDimIdx[tmpVarcharCnt++] = i;
+ } else if (i < sortColCnt && isSortColNoDictFlags[i]) {
noDictSortDimIdx[tmpNoDictSortCnt++] = i;
} else {
noDictNoSortDimIdx[tmpNoDictNoSortCnt++] = i;
@@ -126,10 +144,18 @@ public class TableFieldStat implements Serializable {
return noDictNoSortDimCnt;
}
+ public int getVarcharDimCnt() {
+ return varcharDimCnt;
+ }
+
public boolean[] getIsSortColNoDictFlags() {
return isSortColNoDictFlags;
}
+ public boolean[] getIsVarcharDimFlags() {
+ return isVarcharDimFlags;
+ }
+
public int getMeasureCnt() {
return measureCnt;
}
@@ -154,6 +180,10 @@ public class TableFieldStat implements Serializable {
return noDictNoSortDimIdx;
}
+ public int[] getVarcharDimIdx() {
+ return varcharDimIdx;
+ }
+
public int[] getMeasureIdx() {
return measureIdx;
}
@@ -166,11 +196,12 @@ public class TableFieldStat implements Serializable {
&& dictNoSortDimCnt == that.dictNoSortDimCnt
&& noDictSortDimCnt == that.noDictSortDimCnt
&& noDictNoSortDimCnt == that.noDictNoSortDimCnt
+ && varcharDimCnt == that.varcharDimCnt
&& measureCnt == that.measureCnt;
}
@Override public int hashCode() {
return Objects.hash(dictSortDimCnt, dictNoSortDimCnt, noDictSortDimCnt,
- noDictNoSortDimCnt, measureCnt);
+ noDictNoSortDimCnt, varcharDimCnt, measureCnt);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
index 5408193..1a1c5d1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
@@ -39,7 +39,8 @@ import org.apache.carbondata.core.datastore.page.encoding.EncodedColumnPage;
import org.apache.carbondata.core.datastore.page.encoding.EncodingFactory;
import org.apache.carbondata.core.datastore.page.key.TablePageKey;
import org.apache.carbondata.core.datastore.page.statistics.KeyPageStatsCollector;
-import org.apache.carbondata.core.datastore.page.statistics.LVStringStatsCollector;
+import org.apache.carbondata.core.datastore.page.statistics.LVLongStringStatsCollector;
+import org.apache.carbondata.core.datastore.page.statistics.LVShortStringStatsCollector;
import org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.datastore.row.WriteStepRowUtil;
@@ -98,8 +99,16 @@ public class TablePage {
noDictDimensionPages = new ColumnPage[model.getNoDictionaryCount()];
for (int i = 0; i < noDictDimensionPages.length; i++) {
TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i + numDictDimension);
- ColumnPage page = ColumnPage.newPage(spec, DataTypes.STRING, pageSize);
- page.setStatsCollector(LVStringStatsCollector.newInstance());
+ ColumnPage page;
+ if (DataTypes.VARCHAR == spec.getSchemaDataType()) {
+ page = ColumnPage.newPage(spec, DataTypes.VARCHAR, pageSize);
+ page.setStatsCollector(LVLongStringStatsCollector.newInstance());
+ } else {
+ // In previous implementation, other data types such as string, date and timestamp
+ // will be encoded using string page
+ page = ColumnPage.newPage(spec, DataTypes.STRING, pageSize);
+ page.setStatsCollector(LVShortStringStatsCollector.newInstance());
+ }
noDictDimensionPages[i] = page;
}
complexDimensionPages = new ComplexColumnPage[model.getComplexColumnCount()];
@@ -155,16 +164,21 @@ public class TablePage {
dictDimensionPages[i].putData(rowId, keys[i]);
}
- // 2. convert noDictionary columns and complex columns.
+ // 2. convert noDictionary columns and complex columns and varchar columns.
int noDictionaryCount = noDictDimensionPages.length;
int complexColumnCount = complexDimensionPages.length;
if (noDictionaryCount > 0 || complexColumnCount > 0) {
+ TableSpec tableSpec = model.getTableSpec();
byte[][] noDictAndComplex = WriteStepRowUtil.getNoDictAndComplexDimension(row);
for (int i = 0; i < noDictAndComplex.length; i++) {
- if (i < noDictionaryCount) {
+ if (tableSpec.getDimensionSpec(dictDimensionPages.length + i).getSchemaDataType()
+ == DataTypes.VARCHAR) {
+ byte[] valueWithLength = addIntLengthToByteArray(noDictAndComplex[i]);
+ noDictDimensionPages[i].putData(rowId, valueWithLength);
+ } else if (i < noDictionaryCount) {
// noDictionary columns, since it is variable length, we need to prepare each
// element as LV result byte array (first two bytes are the length of the array)
- byte[] valueWithLength = addLengthToByteArray(noDictAndComplex[i]);
+ byte[] valueWithLength = addShortLengthToByteArray(noDictAndComplex[i]);
noDictDimensionPages[i].putData(rowId, valueWithLength);
} else {
// complex columns
@@ -250,7 +264,7 @@ public class TablePage {
}
// Adds length as a short element (first 2 bytes) to the head of the input byte array
- private byte[] addLengthToByteArray(byte[] input) {
+ private byte[] addShortLengthToByteArray(byte[] input) {
if (input.length > Short.MAX_VALUE) {
throw new RuntimeException("input data length " + input.length +
" bytes too long, maximum length supported is " + Short.MAX_VALUE + " bytes");
@@ -262,6 +276,15 @@ public class TablePage {
return output;
}
+ // Adds length as a integer element (first 4 bytes) to the head of the input byte array
+ private byte[] addIntLengthToByteArray(byte[] input) {
+ byte[] output = new byte[input.length + 4];
+ ByteBuffer buffer = ByteBuffer.wrap(output);
+ buffer.putInt(input.length);
+ buffer.put(input, 0, input.length);
+ return output;
+ }
+
void encode() throws KeyGenException, MemoryException, IOException {
// encode dimensions and measure
EncodedColumnPage[] dimensions = encodeAndCompressDimensions();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index f921fd5..12c95a9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -223,6 +223,26 @@ public final class CarbonDataProcessorUtil {
.toPrimitive(noDictionaryMapping.toArray(new Boolean[noDictionaryMapping.size()]));
}
+ /**
+ * Preparing the boolean [] to map whether the dimension is varchar data type or not.
+ */
+ public static boolean[] getIsVarcharColumnMapping(DataField[] fields) {
+ List<Boolean> isVarcharColumnMapping = new ArrayList<Boolean>();
+ for (DataField field : fields) {
+ // for complex type need to break the loop
+ if (field.getColumn().isComplex()) {
+ break;
+ }
+
+ if (field.getColumn().isDimension()) {
+ isVarcharColumnMapping.add(
+ field.getColumn().getColumnSchema().getDataType() == DataTypes.VARCHAR);
+ }
+ }
+ return ArrayUtils.toPrimitive(
+ isVarcharColumnMapping.toArray(new Boolean[isVarcharColumnMapping.size()]));
+ }
+
public static boolean[] getNoDictionaryMapping(CarbonColumn[] carbonColumns) {
List<Boolean> noDictionaryMapping = new ArrayList<Boolean>();
for (CarbonColumn column : carbonColumns) {
[43/50] [abbrv] carbondata git commit: [HOTFIX] Added Performance
Optimization for Presto by using MultiBlockSplit
Posted by ja...@apache.org.
[HOTFIX] Added Performance Optimization for Presto by using MultiBlockSplit
Added Performance Optimization for Presto by using MultiBlockSplit
This closes #2265
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/01b48fc3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/01b48fc3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/01b48fc3
Branch: refs/heads/carbonstore
Commit: 01b48fc36a93b74827edff4d3330cbf5546a5d38
Parents: 0e1d550
Author: Bhavya <bh...@knoldus.com>
Authored: Mon Apr 16 11:54:17 2018 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Jun 20 22:53:38 2018 +0800
----------------------------------------------------------------------
integration/presto/README.md | 9 +-
integration/presto/pom.xml | 14 +-
.../carbondata/presto/CarbonVectorBatch.java | 2 +-
.../carbondata/presto/CarbondataMetadata.java | 87 +++++++-----
.../carbondata/presto/CarbondataPageSource.java | 11 +-
.../presto/CarbondataPageSourceProvider.java | 27 ++--
.../carbondata/presto/CarbondataSplit.java | 24 +++-
.../presto/CarbondataSplitManager.java | 34 ++++-
.../PrestoCarbonVectorizedRecordReader.java | 39 +++++-
.../presto/impl/CarbonLocalInputSplit.java | 19 ++-
.../presto/impl/CarbonLocalMultiBlockSplit.java | 86 ++++++++++++
.../presto/impl/CarbonTableConfig.java | 40 +++++-
.../presto/impl/CarbonTableReader.java | 133 ++++++++++++-------
.../readers/DecimalSliceStreamReader.java | 105 ++++++++-------
.../presto/src/main/resources/log4j.properties | 11 ++
15 files changed, 462 insertions(+), 179 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/README.md
----------------------------------------------------------------------
diff --git a/integration/presto/README.md b/integration/presto/README.md
index 53884a2..5a44f5c 100644
--- a/integration/presto/README.md
+++ b/integration/presto/README.md
@@ -77,6 +77,9 @@ Please follow the below steps to query carbondata in presto
carbondata-store={schema-store-path}
enable.unsafe.in.query.processing=false
carbon.unsafe.working.memory.in.mb={value}
+ enable.unsafe.columnpage=false
+ enable.unsafe.sort=false
+
```
Replace the schema-store-path with the absolute path of the parent directory of the schema.
For example, if you have a schema named 'default' stored in hdfs://namenode:9000/test/carbondata/,
@@ -112,7 +115,11 @@ Please follow the below steps to query carbondata in presto
#### Unsafe Properties
enable.unsafe.in.query.processing property by default is true in CarbonData system, the carbon.unsafe.working.memory.in.mb
property defines the limit for Unsafe Memory usage in Mega Bytes, the default value is 512 MB.
- If your tables are big you can increase the unsafe memory, or disable unsafe via setting enable.unsafe.in.query.processing=false.
+ Currently Presto does not support Unsafe Memory so we have to disable the unsafe feature by setting below properties to false.
+
+ enable.unsafe.in.query.processing=false.
+ enable.unsafe.columnpage=false
+ enable.unsafe.sort=false
If you updated the jar balls or configuration files, make sure you have dispatched them
to all the presto nodes and restarted the presto servers on the nodes. The updates will not take effect before restarting.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/pom.xml
----------------------------------------------------------------------
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index b91f070..c61023a 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -463,12 +463,6 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- <version>3.0.2</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
<groupId>org.glassfish.hk2</groupId>
<artifactId>hk2-api</artifactId>
<version>2.5.0-b42</version>
@@ -552,13 +546,7 @@
<testSourceDirectory>src/test/scala</testSourceDirectory>
<resources>
<resource>
- <directory>src/resources</directory>
- </resource>
- <resource>
- <directory>.</directory>
- <includes>
- <include>CARBON_SPARK_INTERFACELogResource.properties</include>
- </includes>
+ <directory>src/main/resources</directory>
</resource>
</resources>
<plugins>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java
index b230d6a..b6caaa3 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java
@@ -25,7 +25,7 @@ import org.apache.carbondata.core.scan.result.vector.impl.CarbonColumnVectorImpl
public class CarbonVectorBatch {
- private static final int DEFAULT_BATCH_SIZE = 1024;
+ private static final int DEFAULT_BATCH_SIZE = 4 * 1024;
private final StructField[] schema;
private final int capacity;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
index 8be7494..256e405 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
@@ -17,32 +17,51 @@
package org.apache.carbondata.presto;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
-import org.apache.carbondata.presto.impl.CarbonTableConfig;
-import org.apache.carbondata.presto.impl.CarbonTableReader;
-import com.facebook.presto.spi.*;
-import com.facebook.presto.spi.connector.ConnectorMetadata;
-import com.facebook.presto.spi.type.*;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
+import javax.inject.Inject;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.presto.impl.CarbonTableReader;
-import javax.inject.Inject;
-import java.util.*;
+import com.facebook.presto.spi.ColumnHandle;
+import com.facebook.presto.spi.ColumnMetadata;
+import com.facebook.presto.spi.ConnectorSession;
+import com.facebook.presto.spi.ConnectorTableHandle;
+import com.facebook.presto.spi.ConnectorTableLayout;
+import com.facebook.presto.spi.ConnectorTableLayoutHandle;
+import com.facebook.presto.spi.ConnectorTableLayoutResult;
+import com.facebook.presto.spi.ConnectorTableMetadata;
+import com.facebook.presto.spi.Constraint;
+import com.facebook.presto.spi.SchemaNotFoundException;
+import com.facebook.presto.spi.SchemaTableName;
+import com.facebook.presto.spi.SchemaTablePrefix;
+import com.facebook.presto.spi.connector.ConnectorMetadata;
+import com.facebook.presto.spi.type.BigintType;
+import com.facebook.presto.spi.type.BooleanType;
+import com.facebook.presto.spi.type.DateType;
+import com.facebook.presto.spi.type.DecimalType;
+import com.facebook.presto.spi.type.DoubleType;
+import com.facebook.presto.spi.type.IntegerType;
+import com.facebook.presto.spi.type.SmallintType;
+import com.facebook.presto.spi.type.TimestampType;
+import com.facebook.presto.spi.type.Type;
+import com.facebook.presto.spi.type.VarcharType;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
-import static org.apache.carbondata.presto.Types.checkType;
import static com.google.common.base.Preconditions.checkArgument;
import static java.util.Objects.requireNonNull;
-import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY;
-import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
-import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY;
+import static org.apache.carbondata.presto.Types.checkType;
public class CarbondataMetadata implements ConnectorMetadata {
private final String connectorId;
@@ -119,8 +138,13 @@ public class CarbondataMetadata implements ConnectorMetadata {
List<CarbonColumn> carbonColumns = carbonTable.getCreateOrderColumn(schemaTableName.getTableName());
for (CarbonColumn col : carbonColumns) {
//show columns command will return these data
- Type columnType = carbonDataType2SpiMapper(col.getColumnSchema());
- ColumnMetadata columnMeta = new ColumnMetadata(col.getColumnSchema().getColumnName(), columnType);
+ ColumnSchema columnSchema = col.getColumnSchema();
+ Type columnType = carbonDataType2SpiMapper(columnSchema);
+ String extraValues =
+ columnSchema.getEncodingList().stream().map(encoding -> encoding.toString() + " ")
+ .reduce("", String::concat);
+ ColumnMetadata columnMeta =
+ new ColumnMetadata(columnSchema.getColumnName(), columnType, "", extraValues, false);
columnsMetaList.add(columnMeta);
}
@@ -152,22 +176,22 @@ public class CarbondataMetadata implements ConnectorMetadata {
Type spiType = carbonDataType2SpiMapper(cs);
columnHandles.put(cs.getColumnName(),
- new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType, column.getSchemaOrdinal(),
- column.getKeyOrdinal(), column.getColumnGroupOrdinal(), false, cs.getColumnGroupId(),
- cs.getColumnUniqueId(), cs.isUseInvertedIndex(), cs.getPrecision(), cs.getScale()));
+ new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType,
+ column.getSchemaOrdinal(), column.getKeyOrdinal(), column.getColumnGroupOrdinal(),
+ false, cs.getColumnGroupId(), cs.getColumnUniqueId(), cs.isUseInvertedIndex(),
+ cs.getPrecision(), cs.getScale()));
}
for (CarbonMeasure measure : cb.getMeasureByTableName(tableName)) {
ColumnSchema cs = measure.getColumnSchema();
-
Type spiType = carbonDataType2SpiMapper(cs);
columnHandles.put(cs.getColumnName(),
- new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType, cs.getSchemaOrdinal(),
- measure.getOrdinal(), cs.getColumnGroupId(), true, cs.getColumnGroupId(),
- cs.getColumnUniqueId(), cs.isUseInvertedIndex(), cs.getPrecision(), cs.getScale()));
+ new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType,
+ cs.getSchemaOrdinal(), measure.getOrdinal(), cs.getColumnGroupId(), true,
+ cs.getColumnGroupId(), cs.getColumnUniqueId(), cs.isUseInvertedIndex(),
+ cs.getPrecision(), cs.getScale()));
}
- //should i cache it?
columnHandleMap = columnHandles.build();
return columnHandleMap;
@@ -183,12 +207,7 @@ public class CarbondataMetadata implements ConnectorMetadata {
@Override
public ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) {
- //check tablename is valid
- //schema is exist
- //tables is exist
-
- //CarbondataTable get from jar
- return new CarbondataTableHandle(connectorId, tableName);
+ return new CarbondataTableHandle(connectorId, tableName);
}
@Override public List<ConnectorTableLayoutResult> getTableLayouts(ConnectorSession session,
@@ -196,7 +215,7 @@ public class CarbondataMetadata implements ConnectorMetadata {
Optional<Set<ColumnHandle>> desiredColumns) {
CarbondataTableHandle handle = checkType(table, CarbondataTableHandle.class, "table");
ConnectorTableLayout layout = new ConnectorTableLayout(
- new CarbondataTableLayoutHandle(handle, constraint.getSummary()/*, constraint.getPredicateMap(),constraint.getFilterTuples()*/));
+ new CarbondataTableLayoutHandle(handle, constraint.getSummary()));
return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary()));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
index a7682ce..d31010f 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
@@ -18,7 +18,6 @@
package org.apache.carbondata.presto;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@@ -32,17 +31,13 @@ import com.facebook.presto.hadoop.$internal.com.google.common.base.Throwables;
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ConnectorPageSource;
import com.facebook.presto.spi.Page;
-import com.facebook.presto.spi.PageBuilder;
import com.facebook.presto.spi.PrestoException;
-import com.facebook.presto.spi.RecordCursor;
-import com.facebook.presto.spi.RecordSet;
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.LazyBlock;
import com.facebook.presto.spi.block.LazyBlockLoader;
import com.facebook.presto.spi.type.Type;
import static com.google.common.base.Preconditions.checkState;
-import static java.util.Collections.unmodifiableList;
import static java.util.Objects.requireNonNull;
/**
@@ -53,7 +48,6 @@ class CarbondataPageSource implements ConnectorPageSource {
private static final LogService logger =
LogServiceFactory.getLogService(CarbondataPageSource.class.getName());
private final List<Type> types;
- private final PageBuilder pageBuilder;
private boolean closed;
private PrestoCarbonVectorizedRecordReader vectorReader;
private CarbonDictionaryDecodeReadSupport<Object[]> readSupport;
@@ -69,7 +63,6 @@ class CarbondataPageSource implements ConnectorPageSource {
List<ColumnHandle> columnHandles ) {
this.columnHandles = columnHandles;
this.types = getColumnTypes();
- this.pageBuilder = new PageBuilder(this.types);
this.readSupport = readSupport;
vectorReader = vectorizedRecordReader;
this.readers = createStreamReaders();
@@ -84,7 +77,7 @@ class CarbondataPageSource implements ConnectorPageSource {
}
@Override public boolean isFinished() {
- return closed && pageBuilder.isEmpty();
+ return closed ;
}
@@ -124,7 +117,6 @@ class CarbondataPageSource implements ConnectorPageSource {
blocks[column] = new LazyBlock(batchSize, new CarbondataBlockLoader(column, type));
}
Page page = new Page(batchSize, blocks);
- sizeOfData += columnarBatch.capacity();
return page;
}
catch (PrestoException e) {
@@ -197,6 +189,7 @@ class CarbondataPageSource implements ConnectorPageSource {
checkState(batchId == expectedBatchId);
try {
Block block = readers[columnIndex].readBlock(type);
+ sizeOfData += block.getSizeInBytes();
lazyBlock.setBlock(block);
}
catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
index a268549..4679eac 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
@@ -18,7 +18,6 @@
package org.apache.carbondata.presto;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.common.CarbonIterator;
@@ -31,10 +30,12 @@ import org.apache.carbondata.core.scan.executor.exception.QueryExecutionExceptio
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.scan.result.iterator.AbstractDetailQueryResultIterator;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
import org.apache.carbondata.hadoop.CarbonProjection;
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
-import org.apache.carbondata.presto.impl.CarbonLocalInputSplit;
+import org.apache.carbondata.presto.impl.CarbonLocalMultiBlockSplit;
import org.apache.carbondata.presto.impl.CarbonTableCacheModel;
import org.apache.carbondata.presto.impl.CarbonTableReader;
@@ -64,17 +65,18 @@ public class CarbondataPageSourceProvider implements ConnectorPageSourceProvider
private String connectorId;
private CarbonTableReader carbonTableReader;
+ private String queryId ;
@Inject public CarbondataPageSourceProvider(CarbondataConnectorId connectorId,
CarbonTableReader carbonTableReader) {
this.connectorId = requireNonNull(connectorId, "connectorId is null").toString();
this.carbonTableReader = requireNonNull(carbonTableReader, "carbonTableReader is null");
-
}
@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle,
ConnectorSession session, ConnectorSplit split, List<ColumnHandle> columns) {
+ this.queryId = ((CarbondataSplit)split).getQueryId();
CarbonDictionaryDecodeReadSupport readSupport = new CarbonDictionaryDecodeReadSupport();
PrestoCarbonVectorizedRecordReader carbonRecordReader = createReader(split, columns, readSupport);
return new CarbondataPageSource(readSupport, carbonRecordReader, columns );
@@ -100,8 +102,10 @@ public class CarbondataPageSourceProvider implements ConnectorPageSourceProvider
try {
CarbonIterator iterator = queryExecutor.execute(queryModel);
readSupport.initialize(queryModel.getProjectionColumns(), queryModel.getTable());
- return new PrestoCarbonVectorizedRecordReader(queryExecutor, queryModel,
+ PrestoCarbonVectorizedRecordReader reader = new PrestoCarbonVectorizedRecordReader(queryExecutor, queryModel,
(AbstractDetailQueryResultIterator) iterator);
+ reader.setTaskId(carbondataSplit.getIndex());
+ return reader;
} catch (IOException e) {
throw new RuntimeException("Unable to get the Query Model ", e);
} catch (QueryExecutionException e) {
@@ -129,23 +133,28 @@ public class CarbondataPageSourceProvider implements ConnectorPageSourceProvider
String carbonTablePath = carbonTable.getAbsoluteTableIdentifier().getTablePath();
conf.set(CarbonTableInputFormat.INPUT_DIR, carbonTablePath);
+ conf.set("query.id", queryId);
JobConf jobConf = new JobConf(conf);
CarbonTableInputFormat carbonTableInputFormat = createInputFormat(jobConf, carbonTable,
PrestoFilterUtil.parseFilterExpression(carbondataSplit.getConstraints()),
carbonProjection);
TaskAttemptContextImpl hadoopAttemptContext =
new TaskAttemptContextImpl(jobConf, new TaskAttemptID("", 1, TaskType.MAP, 0, 0));
- CarbonInputSplit carbonInputSplit =
- CarbonLocalInputSplit.convertSplit(carbondataSplit.getLocalInputSplit());
+ CarbonMultiBlockSplit carbonInputSplit =
+ CarbonLocalMultiBlockSplit.convertSplit(carbondataSplit.getLocalInputSplit());
QueryModel queryModel =
carbonTableInputFormat.createQueryModel(carbonInputSplit, hadoopAttemptContext);
+ queryModel.setQueryId(queryId);
queryModel.setVectorReader(true);
+ queryModel.setStatisticsRecorder(
+ CarbonTimeStatisticsFactory.createExecutorRecorder(queryModel.getQueryId()));
- List<CarbonInputSplit> splitList = new ArrayList<>(1);
- splitList.add(carbonInputSplit);
- List<TableBlockInfo> tableBlockInfoList = CarbonInputSplit.createBlocks(splitList);
+ List<TableBlockInfo> tableBlockInfoList =
+ CarbonInputSplit.createBlocks(carbonInputSplit.getAllSplits());
queryModel.setTableBlockInfos(tableBlockInfoList);
+
+
return queryModel;
} catch (IOException e) {
throw new RuntimeException("Unable to get the Query Model ", e);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplit.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplit.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplit.java
index ecc41ef..8a3446b 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplit.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplit.java
@@ -18,6 +18,8 @@
package org.apache.carbondata.presto;
import org.apache.carbondata.presto.impl.CarbonLocalInputSplit;
+import org.apache.carbondata.presto.impl.CarbonLocalMultiBlockSplit;
+
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ConnectorSplit;
import com.facebook.presto.spi.HostAddress;
@@ -36,21 +38,27 @@ public class CarbondataSplit implements ConnectorSplit {
private final String connectorId;
private final SchemaTableName schemaTableName;
private final TupleDomain<ColumnHandle> constraints;
- private final CarbonLocalInputSplit localInputSplit;
+ private final CarbonLocalMultiBlockSplit localInputSplit;
private final List<CarbondataColumnConstraint> rebuildConstraints;
private final ImmutableList<HostAddress> addresses;
+ private final String queryId;
+ private final long index;
@JsonCreator public CarbondataSplit(@JsonProperty("connectorId") String connectorId,
@JsonProperty("schemaTableName") SchemaTableName schemaTableName,
@JsonProperty("constraints") TupleDomain<ColumnHandle> constraints,
- @JsonProperty("localInputSplit") CarbonLocalInputSplit localInputSplit,
- @JsonProperty("rebuildConstraints") List<CarbondataColumnConstraint> rebuildConstraints) {
+ @JsonProperty("localInputSplit") CarbonLocalMultiBlockSplit localInputSplit,
+ @JsonProperty("rebuildConstraints") List<CarbondataColumnConstraint> rebuildConstraints,
+ @JsonProperty("queryId") String queryId,
+ @JsonProperty("index") long index) {
this.connectorId = requireNonNull(connectorId, "connectorId is null");
this.schemaTableName = requireNonNull(schemaTableName, "schemaTable is null");
this.constraints = requireNonNull(constraints, "constraints is null");
this.localInputSplit = requireNonNull(localInputSplit, "localInputSplit is null");
this.rebuildConstraints = requireNonNull(rebuildConstraints, "rebuildConstraints is null");
this.addresses = ImmutableList.of();
+ this.queryId = queryId;
+ this.index = index;
}
@JsonProperty public String getConnectorId() {
@@ -65,7 +73,7 @@ public class CarbondataSplit implements ConnectorSplit {
return constraints;
}
- @JsonProperty public CarbonLocalInputSplit getLocalInputSplit() {
+ @JsonProperty public CarbonLocalMultiBlockSplit getLocalInputSplit() {
return localInputSplit;
}
@@ -84,5 +92,13 @@ public class CarbondataSplit implements ConnectorSplit {
@Override public Object getInfo() {
return this;
}
+
+ @JsonProperty public String getQueryId() {
+ return queryId;
+ }
+
+ @JsonProperty public long getIndex() {
+ return index;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
index 3a54b22..13abd13 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
@@ -22,7 +22,11 @@ import java.util.List;
import java.util.Optional;
import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.presto.impl.CarbonLocalInputSplit;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.presto.impl.CarbonLocalMultiBlockSplit;
import org.apache.carbondata.presto.impl.CarbonTableCacheModel;
import org.apache.carbondata.presto.impl.CarbonTableReader;
@@ -62,7 +66,16 @@ public class CarbondataSplitManager implements ConnectorSplitManager {
CarbondataTableHandle tableHandle = layoutHandle.getTable();
SchemaTableName key = tableHandle.getSchemaTableName();
- // Packaging presto-TupleDomain into CarbondataColumnConstraint, to decouple from presto-spi Module
+ String queryId = System.nanoTime() + "";
+ QueryStatistic statistic = new QueryStatistic();
+ QueryStatisticsRecorder statisticRecorder = CarbonTimeStatisticsFactory.createDriverRecorder();
+ statistic.addStatistics(QueryStatisticsConstants.BLOCK_ALLOCATION, System.currentTimeMillis());
+ statisticRecorder.recordStatisticsForDriver(statistic, queryId);
+ statistic = new QueryStatistic();
+
+ carbonTableReader.setQueryId(queryId);
+ // Packaging presto-TupleDomain into CarbondataColumnConstraint,
+ // to decouple from presto-spi Module
List<CarbondataColumnConstraint> rebuildConstraints =
getColumnConstraints(layoutHandle.getConstraint());
@@ -70,14 +83,23 @@ public class CarbondataSplitManager implements ConnectorSplitManager {
if (null != cache) {
Expression filters = PrestoFilterUtil.parseFilterExpression(layoutHandle.getConstraint());
try {
- List<CarbonLocalInputSplit> splits = carbonTableReader.getInputSplits2(cache, filters,
- layoutHandle.getConstraint());
+ List<CarbonLocalMultiBlockSplit> splits =
+ carbonTableReader.getInputSplits2(cache, filters, layoutHandle.getConstraint());
ImmutableList.Builder<ConnectorSplit> cSplits = ImmutableList.builder();
- for (CarbonLocalInputSplit split : splits) {
+ long index = 0;
+ for (CarbonLocalMultiBlockSplit split : splits) {
+ index++;
cSplits.add(new CarbondataSplit(connectorId, tableHandle.getSchemaTableName(),
- layoutHandle.getConstraint(), split, rebuildConstraints));
+ layoutHandle.getConstraint(), split, rebuildConstraints, queryId, index));
}
+
+ statisticRecorder.logStatisticsAsTableDriver();
+
+ statistic.addStatistics(QueryStatisticsConstants.BLOCK_IDENTIFICATION,
+ System.currentTimeMillis());
+ statisticRecorder.recordStatisticsForDriver(statistic, queryId);
+ statisticRecorder.logStatisticsAsTableDriver();
return new FixedSplitSource(cSplits.build());
} catch (Exception ex) {
throw new RuntimeException(ex.getMessage(), ex);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
index a1907db..913d423 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
@@ -39,6 +39,10 @@ import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.scan.result.iterator.AbstractDetailQueryResultIterator;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+import org.apache.carbondata.core.stats.TaskStatistics;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.hadoop.AbstractRecordReader;
import org.apache.carbondata.hadoop.CarbonInputSplit;
@@ -72,11 +76,17 @@ class PrestoCarbonVectorizedRecordReader extends AbstractRecordReader<Object> {
private QueryExecutor queryExecutor;
- public PrestoCarbonVectorizedRecordReader(QueryExecutor queryExecutor, QueryModel queryModel, AbstractDetailQueryResultIterator iterator) {
+ private long taskId;
+
+ private long queryStartTime;
+
+ public PrestoCarbonVectorizedRecordReader(QueryExecutor queryExecutor, QueryModel queryModel,
+ AbstractDetailQueryResultIterator iterator) {
this.queryModel = queryModel;
this.iterator = iterator;
this.queryExecutor = queryExecutor;
enableReturningBatches();
+ this.queryStartTime = System.currentTimeMillis();
}
/**
@@ -125,6 +135,8 @@ class PrestoCarbonVectorizedRecordReader extends AbstractRecordReader<Object> {
} catch (QueryExecutionException e) {
throw new IOException(e);
}
+
+ logStatistics(taskId, queryStartTime, queryModel.getStatisticsRecorder());
}
@Override public boolean nextKeyValue() throws IOException, InterruptedException {
@@ -239,5 +251,30 @@ class PrestoCarbonVectorizedRecordReader extends AbstractRecordReader<Object> {
return false;
}
+ public void setTaskId(long taskId) {
+ this.taskId = taskId;
+ }
+
+ /**
+ * For Logging the Statistics
+ * @param taskId
+ * @param queryStartTime
+ * @param recorder
+ */
+ private void logStatistics(
+ Long taskId,
+ Long queryStartTime,
+ QueryStatisticsRecorder recorder
+ ) {
+ if (null != recorder) {
+ QueryStatistic queryStatistic = new QueryStatistic();
+ queryStatistic.addFixedTimeStatistic(QueryStatisticsConstants.EXECUTOR_PART,
+ System.currentTimeMillis() - queryStartTime);
+ recorder.recordStatistics(queryStatistic);
+ // print executor query statistics for each task_id
+ TaskStatistics statistics = recorder.statisticsForTask(taskId, queryStartTime);
+ recorder.logStatisticsForTask(statistics);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
index 2c6a810..af23671 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
@@ -42,6 +42,7 @@ public class CarbonLocalInputSplit {
private List<String> locations;// locations are the locations for different replicas.
private short version;
private String[] deleteDeltaFiles;
+ private String blockletId;
private String detailInfo;
@@ -87,6 +88,10 @@ public class CarbonLocalInputSplit {
return detailInfo;
}
+ @JsonProperty public String getBlockletId() {
+ return blockletId;
+ }
+
public void setDetailInfo(BlockletDetailInfo blockletDetailInfo) {
Gson gson = new Gson();
detailInfo = gson.toJson(blockletDetailInfo);
@@ -100,6 +105,7 @@ public class CarbonLocalInputSplit {
@JsonProperty("tableBlockInfo") TableBlockInfo tableBlockInfo*/,
@JsonProperty("version") short version,
@JsonProperty("deleteDeltaFiles") String[] deleteDeltaFiles,
+ @JsonProperty("blockletId") String blockletId,
@JsonProperty("detailInfo") String detailInfo
) {
this.path = path;
@@ -111,22 +117,23 @@ public class CarbonLocalInputSplit {
//this.tableBlockInfo = tableBlockInfo;
this.version = version;
this.deleteDeltaFiles = deleteDeltaFiles;
+ this.blockletId = blockletId;
this.detailInfo = detailInfo;
}
public static CarbonInputSplit convertSplit(CarbonLocalInputSplit carbonLocalInputSplit) {
- CarbonInputSplit inputSplit = new CarbonInputSplit(carbonLocalInputSplit.getSegmentId(), "0",
- new Path(carbonLocalInputSplit.getPath()), carbonLocalInputSplit.getStart(),
- carbonLocalInputSplit.getLength(), carbonLocalInputSplit.getLocations()
- .toArray(new String[carbonLocalInputSplit.getLocations().size()]),
+ CarbonInputSplit inputSplit = new CarbonInputSplit(carbonLocalInputSplit.getSegmentId(),
+ carbonLocalInputSplit.getBlockletId(), new Path(carbonLocalInputSplit.getPath()),
+ carbonLocalInputSplit.getStart(), carbonLocalInputSplit.getLength(),
+ carbonLocalInputSplit.getLocations()
+ .toArray(new String[carbonLocalInputSplit.getLocations().size()]),
carbonLocalInputSplit.getNumberOfBlocklets(),
ColumnarFormatVersion.valueOf(carbonLocalInputSplit.getVersion()),
carbonLocalInputSplit.getDeleteDeltaFiles());
Gson gson = new Gson();
BlockletDetailInfo blockletDetailInfo =
gson.fromJson(carbonLocalInputSplit.detailInfo, BlockletDetailInfo.class);
-
if (null == blockletDetailInfo) {
throw new RuntimeException("Could not read blocklet details");
}
@@ -138,4 +145,6 @@ public class CarbonLocalInputSplit {
inputSplit.setDetailInfo(blockletDetailInfo);
return inputSplit;
}
+
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalMultiBlockSplit.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalMultiBlockSplit.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalMultiBlockSplit.java
new file mode 100755
index 0000000..37174c1
--- /dev/null
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalMultiBlockSplit.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.presto.impl;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.carbondata.core.statusmanager.FileFormat;
+import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * CarbonLocalInputSplit represents a block, it contains a set of blocklet.
+ */
+public class CarbonLocalMultiBlockSplit {
+
+ private static final long serialVersionUID = 3520344046772190207L;
+
+ /*
+ * Splits (HDFS Blocks) for task to scan.
+ */
+ private List<CarbonLocalInputSplit> splitList;
+
+ /*
+ * The locations of all wrapped splits
+ */
+ private String[] locations;
+
+ private FileFormat fileFormat = FileFormat.COLUMNAR_V3;
+
+ private long length;
+
+ @JsonProperty public long getLength() {
+ return length;
+ }
+
+ @JsonProperty public String[] getLocations() {
+ return locations;
+ }
+
+ @JsonProperty public List<CarbonLocalInputSplit> getSplitList() {
+ return splitList;
+ }
+
+ @JsonProperty public FileFormat getFileFormat() {
+ return fileFormat;
+ }
+
+ @JsonCreator public CarbonLocalMultiBlockSplit(
+ @JsonProperty("splitList") List<CarbonLocalInputSplit> splitList,
+ @JsonProperty("locations") String[] locations) {
+ this.splitList = splitList;
+ this.locations = locations;
+ }
+
+ public static CarbonMultiBlockSplit convertSplit(
+ CarbonLocalMultiBlockSplit carbonLocalMultiBlockSplit) {
+ List<CarbonInputSplit> carbonInputSplitList =
+ carbonLocalMultiBlockSplit.getSplitList().stream().map(CarbonLocalInputSplit::convertSplit)
+ .collect(Collectors.toList());
+
+ CarbonMultiBlockSplit carbonMultiBlockSplit =
+ new CarbonMultiBlockSplit(carbonInputSplitList, carbonLocalMultiBlockSplit.getLocations());
+
+ return carbonMultiBlockSplit;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
index 75a7f11..f800e59 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
@@ -17,10 +17,10 @@
package org.apache.carbondata.presto.impl;
-import javax.validation.constraints.NotNull;
-
import io.airlift.configuration.Config;
+import javax.validation.constraints.NotNull;
+
/**
* Configuration read from etc/catalog/carbondata.properties
*/
@@ -32,6 +32,10 @@ public class CarbonTableConfig {
private String storePath;
private String unsafeMemoryInMb;
private String enableUnsafeInQueryExecution;
+ private String enableUnsafeColumnPage;
+ private String enableUnsafeSort;
+ private String enableQueryStatistics;
+ private String batchSize;
private String s3A_acesssKey;
private String s3A_secretKey;
private String s3_acesssKey;
@@ -88,6 +92,38 @@ public class CarbonTableConfig {
return this;
}
+ public String getEnableUnsafeColumnPage() { return enableUnsafeColumnPage; }
+
+ @Config("enable.unsafe.columnpage")
+ public CarbonTableConfig setEnableUnsafeColumnPage(String enableUnsafeColumnPage) {
+ this.enableUnsafeColumnPage = enableUnsafeColumnPage;
+ return this;
+ }
+
+ public String getEnableUnsafeSort() { return enableUnsafeSort; }
+
+ @Config("enable.unsafe.sort")
+ public CarbonTableConfig setEnableUnsafeSort(String enableUnsafeSort) {
+ this.enableUnsafeSort = enableUnsafeSort;
+ return this;
+ }
+
+ public String getEnableQueryStatistics() { return enableQueryStatistics; }
+
+ @Config("enable.query.statistics")
+ public CarbonTableConfig setEnableQueryStatistics(String enableQueryStatistics) {
+ this.enableQueryStatistics = enableQueryStatistics;
+ return this;
+ }
+
+ public String getBatchSize() { return batchSize; }
+
+ @Config("query.vector.batchSize")
+ public CarbonTableConfig setBatchSize(String batchSize) {
+ this.batchSize = batchSize;
+ return this;
+ }
+
public String getS3A_AcesssKey() {
return s3A_acesssKey;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
index 5866ad1..77c8ce5 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
@@ -121,6 +121,8 @@ public class CarbonTableReader {
private LoadMetadataDetails loadMetadataDetails[];
+ private String queryId;
+
/**
* Logger instance
*/
@@ -137,6 +139,7 @@ public class CarbonTableReader {
this.carbonCache = new AtomicReference(new HashMap());
tableList = new ConcurrentSet<>();
setS3Properties();
+ populateCarbonProperties();
}
/**
@@ -218,10 +221,11 @@ public class CarbonTableReader {
} else return ImmutableList.of();
}
- private void getName(CarbonFile carbonFile){
- if(!carbonFile.getName().equalsIgnoreCase("_system") && !carbonFile.getName().equalsIgnoreCase(".ds_store")){
- schemaNames.add(carbonFile.getName());
- }
+ private void getName(CarbonFile carbonFile) {
+ if (!carbonFile.getName().equalsIgnoreCase("_system") && !carbonFile.getName()
+ .equalsIgnoreCase(".ds_store")) {
+ schemaNames.add(carbonFile.getName());
+ }
}
/**
@@ -243,11 +247,11 @@ public class CarbonTableReader {
*/
private Set<String> updateTableList(String schemaName) {
List<CarbonFile> schema =
- Stream.of(carbonFileList.listFiles()).filter(a -> schemaName.equals(a.getName()))
- .collect(Collectors.toList());
+ Stream.of(carbonFileList.listFiles()).filter(a -> schemaName.equals(a.getName()))
+ .collect(Collectors.toList());
if (schema.size() > 0) {
return Stream.of((schema.get(0)).listFiles()).map(CarbonFile::getName)
- .collect(Collectors.toSet());
+ .collect(Collectors.toSet());
} else return ImmutableSet.of();
}
@@ -294,11 +298,11 @@ public class CarbonTableReader {
if (isKeyExists) {
CarbonTableCacheModel carbonTableCacheModel = carbonCache.get().get(schemaTableName);
- if(carbonTableCacheModel != null && carbonTableCacheModel.carbonTable.getTableInfo() != null) {
- Long latestTime = FileFactory.getCarbonFile(
- CarbonTablePath.getSchemaFilePath(
- carbonCache.get().get(schemaTableName).carbonTable.getTablePath())
- ).getLastModifiedTime();
+ if (carbonTableCacheModel != null
+ && carbonTableCacheModel.carbonTable.getTableInfo() != null) {
+ Long latestTime = FileFactory.getCarbonFile(CarbonTablePath
+ .getSchemaFilePath(carbonCache.get().get(schemaTableName).carbonTable.getTablePath()))
+ .getLastModifiedTime();
Long oldTime = carbonTableCacheModel.carbonTable.getTableInfo().getLastUpdatedTime();
if (DateUtils.truncate(new Date(latestTime), Calendar.MINUTE)
.after(DateUtils.truncate(new Date(oldTime), Calendar.MINUTE))) {
@@ -317,7 +321,6 @@ public class CarbonTableReader {
}
}
-
/**
* Find the table with the given name and build a CarbonTable instance for it.
* This method should be called after this.updateSchemaTables().
@@ -401,19 +404,10 @@ public class CarbonTableReader {
return result;
}
- public List<CarbonLocalInputSplit> getInputSplits2(CarbonTableCacheModel tableCacheModel,
+ public List<CarbonLocalMultiBlockSplit> getInputSplits2(CarbonTableCacheModel tableCacheModel,
Expression filters, TupleDomain<ColumnHandle> constraints) throws IOException {
List<CarbonLocalInputSplit> result = new ArrayList<>();
- if(config.getUnsafeMemoryInMb() != null) {
- CarbonProperties.getInstance().addProperty(
- CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
- config.getUnsafeMemoryInMb());
- }
- if(config.getEnableUnsafeInQueryExecution() != null) {
- CarbonProperties.getInstance().addProperty(
- CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION,
- config.getEnableUnsafeInQueryExecution());
- }
+ List<CarbonLocalMultiBlockSplit> multiBlockSplitList = new ArrayList<>();
CarbonTable carbonTable = tableCacheModel.carbonTable;
TableInfo tableInfo = tableCacheModel.carbonTable.getTableInfo();
Configuration config = new Configuration();
@@ -422,56 +416,73 @@ public class CarbonTableReader {
config.set(CarbonTableInputFormat.INPUT_DIR, carbonTablePath);
config.set(CarbonTableInputFormat.DATABASE_NAME, carbonTable.getDatabaseName());
config.set(CarbonTableInputFormat.TABLE_NAME, carbonTable.getTableName());
+ config.set("query.id", queryId);
JobConf jobConf = new JobConf(config);
List<PartitionSpec> filteredPartitions = new ArrayList();
PartitionInfo partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName());
- if(partitionInfo!=null && partitionInfo.getPartitionType()== PartitionType.NATIVE_HIVE) {
+ if (partitionInfo != null && partitionInfo.getPartitionType() == PartitionType.NATIVE_HIVE) {
try {
- loadMetadataDetails= SegmentStatusManager
- .readTableStatusFile(CarbonTablePath.getTableStatusFilePath(carbonTable.getTablePath()));
+ loadMetadataDetails = SegmentStatusManager.readTableStatusFile(
+ CarbonTablePath.getTableStatusFilePath(carbonTable.getTablePath()));
} catch (IOException exception) {
LOGGER.error(exception.getMessage());
throw exception;
}
- filteredPartitions = findRequiredPartitions(constraints, carbonTable,loadMetadataDetails);
+ filteredPartitions = findRequiredPartitions(constraints, carbonTable, loadMetadataDetails);
}
try {
CarbonTableInputFormat.setTableInfo(config, tableInfo);
CarbonTableInputFormat carbonTableInputFormat =
- createInputFormat(jobConf, carbonTable.getAbsoluteTableIdentifier(), filters,filteredPartitions);
+ createInputFormat(jobConf, carbonTable.getAbsoluteTableIdentifier(), filters,
+ filteredPartitions);
Job job = Job.getInstance(jobConf);
List<InputSplit> splits = carbonTableInputFormat.getSplits(job);
- CarbonInputSplit carbonInputSplit = null;
Gson gson = new Gson();
if (splits != null && splits.size() > 0) {
for (InputSplit inputSplit : splits) {
- carbonInputSplit = (CarbonInputSplit) inputSplit;
+ CarbonInputSplit carbonInputSplit = (CarbonInputSplit) inputSplit;
result.add(new CarbonLocalInputSplit(carbonInputSplit.getSegmentId(),
carbonInputSplit.getPath().toString(), carbonInputSplit.getStart(),
carbonInputSplit.getLength(), Arrays.asList(carbonInputSplit.getLocations()),
carbonInputSplit.getNumberOfBlocklets(), carbonInputSplit.getVersion().number(),
- carbonInputSplit.getDeleteDeltaFiles(),
+ carbonInputSplit.getDeleteDeltaFiles(), carbonInputSplit.getBlockletId(),
gson.toJson(carbonInputSplit.getDetailInfo())));
}
+
+ // Use block distribution
+ List<List<CarbonLocalInputSplit>> inputSplits = new ArrayList(
+ result.stream().map(x -> (CarbonLocalInputSplit) x).collect(Collectors.groupingBy(
+ carbonInput -> carbonInput.getSegmentId().concat(carbonInput.getPath()))).values());
+ if (inputSplits != null) {
+ for (int j = 0; j < inputSplits.size(); j++) {
+ multiBlockSplitList.add(new CarbonLocalMultiBlockSplit(inputSplits.get(j),
+ inputSplits.get(j).stream().flatMap(f -> Arrays.stream(getLocations(f))).distinct()
+ .toArray(String[]::new)));
+ }
+ }
+ LOGGER.error("Size fo MultiblockList " + multiBlockSplitList.size());
+
}
} catch (IOException e) {
throw new RuntimeException("Error creating Splits from CarbonTableInputFormat", e);
}
- return result;
+ return multiBlockSplitList;
}
- /** Returns list of partition specs to query based on the domain constraints
+ /**
+ * Returns list of partition specs to query based on the domain constraints
+ *
* @param constraints
* @param carbonTable
* @throws IOException
*/
- private List<PartitionSpec> findRequiredPartitions(TupleDomain<ColumnHandle> constraints, CarbonTable carbonTable,
- LoadMetadataDetails[]loadMetadataDetails) throws IOException {
+ private List<PartitionSpec> findRequiredPartitions(TupleDomain<ColumnHandle> constraints,
+ CarbonTable carbonTable, LoadMetadataDetails[] loadMetadataDetails) throws IOException {
Set<PartitionSpec> partitionSpecs = new HashSet<>();
List<PartitionSpec> prunePartitions = new ArrayList();
@@ -490,29 +501,40 @@ public class CarbonTableReader {
List<String> partitionValuesFromExpression =
PrestoFilterUtil.getPartitionFilters(carbonTable, constraints);
- List<PartitionSpec> partitionSpecList = partitionSpecs.stream().filter( partitionSpec ->
- CollectionUtils.isSubCollection(partitionValuesFromExpression, partitionSpec.getPartitions())).collect(Collectors.toList());
+ List<PartitionSpec> partitionSpecList = partitionSpecs.stream().filter(
+ partitionSpec -> CollectionUtils
+ .isSubCollection(partitionValuesFromExpression, partitionSpec.getPartitions()))
+ .collect(Collectors.toList());
prunePartitions.addAll(partitionSpecList);
return prunePartitions;
}
- private CarbonTableInputFormat<Object> createInputFormat( Configuration conf,
- AbsoluteTableIdentifier identifier, Expression filterExpression, List<PartitionSpec> filteredPartitions)
- throws IOException {
+ private CarbonTableInputFormat<Object> createInputFormat(Configuration conf,
+ AbsoluteTableIdentifier identifier, Expression filterExpression,
+ List<PartitionSpec> filteredPartitions) throws IOException {
CarbonTableInputFormat format = new CarbonTableInputFormat<Object>();
- CarbonTableInputFormat.setTablePath(conf,
- identifier.appendWithLocalPrefix(identifier.getTablePath()));
+ CarbonTableInputFormat
+ .setTablePath(conf, identifier.appendWithLocalPrefix(identifier.getTablePath()));
CarbonTableInputFormat.setFilterPredicates(conf, filterExpression);
- if(filteredPartitions.size() != 0) {
+ if (filteredPartitions.size() != 0) {
CarbonTableInputFormat.setPartitionsToPrune(conf, filteredPartitions);
}
return format;
}
+ private void populateCarbonProperties() {
+ addProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, config.getUnsafeMemoryInMb());
+ addProperty(CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION,
+ config.getEnableUnsafeInQueryExecution());
+ addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE,
+ config.getEnableUnsafeColumnPage());
+ addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, config.getEnableUnsafeSort());
+ addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, config.getEnableQueryStatistics());
+ }
private void setS3Properties(){
- FileFactory.getConfiguration()
+ FileFactory.getConfiguration()
.set(ACCESS_KEY, Objects.toString(config.getS3A_AcesssKey(),""));
FileFactory.getConfiguration()
.set(SECRET_KEY, Objects.toString(config.getS3A_SecretKey()));
@@ -528,4 +550,25 @@ public class CarbonTableReader {
Objects.toString(config.getS3EndPoint(),""));
}
+ private void addProperty(String propertyName, String propertyValue) {
+ if (propertyValue != null) {
+ CarbonProperties.getInstance().addProperty(propertyName, propertyValue);
+ }
+ }
+
+ /**
+ * @param cis
+ * @return
+ */
+ private String[] getLocations(CarbonLocalInputSplit cis) {
+ return cis.getLocations().toArray(new String[cis.getLocations().size()]);
+ }
+
+ public String getQueryId() {
+ return queryId;
+ }
+
+ public void setQueryId(String queryId) {
+ this.queryId = queryId;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java
index e6ac386..54f2b5f 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DecimalSliceStreamReader.java
@@ -68,32 +68,36 @@ public class DecimalSliceStreamReader extends AbstractStreamReader {
* @return
* @throws IOException
*/
- public Block readBlock(Type type)
- throws IOException
- {
+ public Block readBlock(Type type) throws IOException {
int numberOfRows = 0;
BlockBuilder builder = null;
- if(isVectorReader) {
+ if (isVectorReader) {
numberOfRows = batchSize;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
if (columnVector != null) {
- if(columnVector.anyNullsSet())
- {
- handleNullInVector(type, numberOfRows, builder);
+ if (isDictionary) {
+ if (isShortDecimal(type)) {
+ populateShortDictionaryVector(type, numberOfRows, builder);
+ } else {
+ populateLongDictionaryVector(type, numberOfRows, builder);
+ }
} else {
- if(isShortDecimal(type)) {
- populateShortDecimalVector(type, numberOfRows, builder);
+ if (columnVector.anyNullsSet()) {
+ handleNullInVector(type, numberOfRows, builder);
} else {
- populateLongDecimalVector(type, numberOfRows, builder);
+ if (isShortDecimal(type)) {
+ populateShortDecimalVector(type, numberOfRows, builder);
+ } else {
+ populateLongDecimalVector(type, numberOfRows, builder);
+ }
}
}
}
-
} else {
if (streamData != null) {
numberOfRows = streamData.length;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
- for(int i = 0; i < numberOfRows ; i++ ){
+ for (int i = 0; i < numberOfRows; i++) {
Slice slice = getSlice(streamData[i], type);
if (isShortDecimal(type)) {
type.writeLong(builder, parseLong((DecimalType) type, slice, 0, slice.length()));
@@ -211,52 +215,55 @@ public class DecimalSliceStreamReader extends AbstractStreamReader {
private void populateShortDecimalVector(Type type, int numberOfRows, BlockBuilder builder) {
DecimalType decimalType = (DecimalType) type;
+ for (int i = 0; i < numberOfRows; i++) {
+ BigDecimal decimalValue = (BigDecimal) columnVector.getData(i);
+ long rescaledDecimal = Decimals
+ .rescale(decimalValue.unscaledValue().longValue(), decimalValue.scale(),
+ decimalType.getScale());
+ type.writeLong(builder, rescaledDecimal);
+ }
+ }
- if (isDictionary) {
- for (int i = 0; i < numberOfRows; i++) {
- int value = (int)columnVector.getData(i);
- Object data = DataTypeUtil
- .getDataBasedOnDataType(dictionary.getDictionaryValueForKey(value), DataTypes.createDecimalType(decimalType.getPrecision(), decimalType.getScale()));
- if(Objects.isNull(data)) {
- builder.appendNull();
- } else {
- BigDecimal decimalValue = (BigDecimal) data;
- long rescaledDecimal =
- Decimals.rescale(decimalValue.unscaledValue().longValue(), decimalValue.scale(),decimalType.getScale());
- type.writeLong(builder, rescaledDecimal);
- }
- }
- } else {
- for (int i = 0; i < numberOfRows; i++) {
- BigDecimal decimalValue = (BigDecimal) columnVector.getData(i);
- long rescaledDecimal =
- Decimals.rescale(decimalValue.unscaledValue().longValue(), decimalValue.scale(),decimalType.getScale());
+ private void populateLongDecimalVector(Type type, int numberOfRows, BlockBuilder builder) {
+ for (int i = 0; i < numberOfRows; i++) {
+ Slice slice = getSlice((columnVector.getData(i)), type);
+ type.writeSlice(builder, parseSlice((DecimalType) type, slice, 0, slice.length()));
+ }
+ }
+
+ private void populateShortDictionaryVector(Type type, int numberOfRows, BlockBuilder builder) {
+ DecimalType decimalType = (DecimalType) type;
+ for (int i = 0; i < numberOfRows; i++) {
+ int value = (int) columnVector.getData(i);
+ Object data = DataTypeUtil.getDataBasedOnDataType(dictionary.getDictionaryValueForKey(value),
+ DataTypes.createDecimalType(decimalType.getPrecision(), decimalType.getScale()));
+ if (Objects.isNull(data)) {
+ builder.appendNull();
+ } else {
+ BigDecimal decimalValue = (BigDecimal) data;
+ long rescaledDecimal = Decimals
+ .rescale(decimalValue.unscaledValue().longValue(), decimalValue.scale(),
+ decimalType.getScale());
type.writeLong(builder, rescaledDecimal);
}
}
}
- private void populateLongDecimalVector(Type type, int numberOfRows, BlockBuilder builder) {
- if (isDictionary) {
- for (int i = 0; i < numberOfRows; i++) {
- int value = (int) columnVector.getData(i);
- DecimalType decimalType = (DecimalType) type;
- Object data = DataTypeUtil
- .getDataBasedOnDataType(dictionary.getDictionaryValueForKey(value), DataTypes.createDecimalType(decimalType.getPrecision(), decimalType.getScale()));
- if(Objects.isNull(data)) {
- builder.appendNull();
- } else {
- BigDecimal decimalValue = (BigDecimal) data;
- Slice slice = getSlice(decimalValue, type);
- type.writeSlice(builder, parseSlice((DecimalType) type, slice, 0, slice.length()));
- }
- }
- } else {
- for (int i = 0; i < numberOfRows; i++) {
- Slice slice = getSlice((columnVector.getData(i)), type);
+ private void populateLongDictionaryVector(Type type, int numberOfRows, BlockBuilder builder) {
+ DecimalType decimalType = (DecimalType) type;
+ for (int i = 0; i < numberOfRows; i++) {
+ int value = (int) columnVector.getData(i);
+ Object data = DataTypeUtil.getDataBasedOnDataType(dictionary.getDictionaryValueForKey(value),
+ DataTypes.createDecimalType(decimalType.getPrecision(), decimalType.getScale()));
+ if (Objects.isNull(data)) {
+ builder.appendNull();
+ } else {
+ BigDecimal decimalValue = (BigDecimal) data;
+ Slice slice = getSlice(decimalValue, type);
type.writeSlice(builder, parseSlice((DecimalType) type, slice, 0, slice.length()));
}
}
}
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/resources/log4j.properties b/integration/presto/src/main/resources/log4j.properties
new file mode 100644
index 0000000..e369916
--- /dev/null
+++ b/integration/presto/src/main/resources/log4j.properties
@@ -0,0 +1,11 @@
+# Root logger option
+log4j.rootLogger=INFO,stdout
+
+
+# Redirect log messages to console
+log4j.appender.debug=org.apache.log4j.RollingFileAppender
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
[49/50] [abbrv] carbondata git commit: [CARBONDATA-2513][32K] Support
write long string from dataframe
Posted by ja...@apache.org.
[CARBONDATA-2513][32K] Support write long string from dataframe
support write long string from dataframe
This closes #2382
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/55f4bc6c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/55f4bc6c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/55f4bc6c
Branch: refs/heads/carbonstore
Commit: 55f4bc6c89f637b162b414033512901e9bd8a745
Parents: 218a8de
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Wed Jun 20 19:01:24 2018 +0800
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Thu Jun 21 12:31:21 2018 +0530
----------------------------------------------------------------------
.../VarcharDataTypesBasicTestCase.scala | 32 +++++++++++++++++++-
.../apache/carbondata/spark/CarbonOption.scala | 2 ++
.../spark/rdd/NewCarbonDataLoadRDD.scala | 15 +++++++--
.../carbondata/spark/util/CarbonScalaUtil.scala | 3 +-
.../spark/sql/CarbonDataFrameWriter.scala | 1 +
.../streaming/parser/FieldConverter.scala | 11 +++++--
6 files changed, 57 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55f4bc6c/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
index 9ea3f1f..9798178 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
@@ -20,8 +20,9 @@ package org.apache.carbondata.spark.testsuite.longstring
import java.io.{File, PrintWriter}
import org.apache.commons.lang3.RandomStringUtils
-import org.apache.spark.sql.Row
+import org.apache.spark.sql.{DataFrame, Row, SaveMode}
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -36,6 +37,7 @@ class VarcharDataTypesBasicTestCase extends QueryTest with BeforeAndAfterEach wi
private val inputFile_2g_column_page = s"$inputDir$fileName_2g_column_page"
private val lineNum = 1000
private var content: Content = _
+ private var longStringDF: DataFrame = _
private var originMemorySize = CarbonProperties.getInstance().getProperty(
CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT)
@@ -257,6 +259,34 @@ class VarcharDataTypesBasicTestCase extends QueryTest with BeforeAndAfterEach wi
// since after exception wrapper, we cannot get the root cause directly
}
+ private def prepareDF(): Unit = {
+ val schema = StructType(
+ StructField("id", IntegerType, nullable = true) ::
+ StructField("name", StringType, nullable = true) ::
+ StructField("description", StringType, nullable = true) ::
+ StructField("address", StringType, nullable = true) ::
+ StructField("note", StringType, nullable = true) :: Nil
+ )
+ longStringDF = sqlContext.sparkSession.read
+ .schema(schema)
+ .csv(inputFile)
+ }
+
+ test("write from dataframe with long string datatype") {
+ prepareDF()
+ // write spark dataframe to carbondata with `long_string_columns` property
+ longStringDF.write
+ .format("carbondata")
+ .option("tableName", longStringTable)
+ .option("single_pass", "false")
+ .option("sort_columns", "name")
+ .option("long_string_columns", "description, note")
+ .mode(SaveMode.Overwrite)
+ .save()
+
+ checkQuery()
+ }
+
// will create 2 long string columns
private def createFile(filePath: String, line: Int = 10000, start: Int = 0,
varcharLen: Int = Short.MaxValue + 1000): Content = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55f4bc6c/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
index a48e63d..5f23f77 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
@@ -48,6 +48,8 @@ class CarbonOption(options: Map[String, String]) {
def dictionaryExclude: Option[String] = options.get("dictionary_exclude")
+ def longStringColumns: Option[String] = options.get("long_string_columns")
+
def tableBlockSize: Option[String] = options.get("table_blocksize")
def bucketNumber: Int = options.getOrElse("bucketnumber", "0").toInt
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55f4bc6c/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 4bfdd3b..5ed39fa 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -41,6 +41,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.common.logging.impl.StandardLogService
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.compression.CompressorFactory
+import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, ThreadLocalTaskInfo}
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -447,6 +448,10 @@ class NewRddIterator(rddIter: Iterator[Row],
private val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
private val serializationNullFormat =
carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
+ import scala.collection.JavaConverters._
+ private val isVarcharTypeMapping =
+ carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getCreateOrderColumn(
+ carbonLoadModel.getTableName).asScala.map(_.getDataType == DataTypes.VARCHAR)
def hasNext: Boolean = rddIter.hasNext
def next: Array[AnyRef] = {
@@ -454,7 +459,8 @@ class NewRddIterator(rddIter: Iterator[Row],
val columns = new Array[AnyRef](row.length)
for (i <- 0 until columns.length) {
columns(i) = CarbonScalaUtil.getString(row.get(i), serializationNullFormat,
- delimiterLevel1, delimiterLevel2, timeStampFormat, dateFormat)
+ delimiterLevel1, delimiterLevel2, timeStampFormat, dateFormat,
+ isVarcharType = i < isVarcharTypeMapping.size && isVarcharTypeMapping(i))
}
columns
}
@@ -491,6 +497,10 @@ class LazyRddIterator(serializer: SerializerInstance,
private val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
private val serializationNullFormat =
carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
+ import scala.collection.JavaConverters._
+ private val isVarcharTypeMapping =
+ carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getCreateOrderColumn(
+ carbonLoadModel.getTableName).asScala.map(_.getDataType == DataTypes.VARCHAR)
private var rddIter: Iterator[Row] = null
private var uninitialized = true
@@ -514,7 +524,8 @@ class LazyRddIterator(serializer: SerializerInstance,
val columns = new Array[AnyRef](row.length)
for (i <- 0 until columns.length) {
columns(i) = CarbonScalaUtil.getString(row.get(i), serializationNullFormat,
- delimiterLevel1, delimiterLevel2, timeStampFormat, dateFormat)
+ delimiterLevel1, delimiterLevel2, timeStampFormat, dateFormat,
+ isVarcharType = i < isVarcharTypeMapping.size && isVarcharTypeMapping(i))
}
columns
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55f4bc6c/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 3e94a66..44d3cca 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -127,9 +127,10 @@ object CarbonScalaUtil {
delimiterLevel2: String,
timeStampFormat: SimpleDateFormat,
dateFormat: SimpleDateFormat,
+ isVarcharType: Boolean = false,
level: Int = 1): String = {
FieldConverter.objectToString(value, serializationNullFormat, delimiterLevel1,
- delimiterLevel2, timeStampFormat, dateFormat, level)
+ delimiterLevel2, timeStampFormat, dateFormat, isVarcharType = isVarcharType, level)
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55f4bc6c/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
index 67817c0..c81622e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
@@ -86,6 +86,7 @@ class CarbonDataFrameWriter(sqlContext: SQLContext, val dataFrame: DataFrame) {
"SORT_COLUMNS" -> options.sortColumns,
"DICTIONARY_INCLUDE" -> options.dictionaryInclude,
"DICTIONARY_EXCLUDE" -> options.dictionaryExclude,
+ "LONG_STRING_COLUMNS" -> options.longStringColumns,
"TABLE_BLOCKSIZE" -> options.tableBlockSize,
"STREAMING" -> Option(options.isStreaming.toString)
).filter(_._2.isDefined)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55f4bc6c/streaming/src/main/scala/org/apache/carbondata/streaming/parser/FieldConverter.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/parser/FieldConverter.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/parser/FieldConverter.scala
index 8661417..e167d46 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/parser/FieldConverter.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/parser/FieldConverter.scala
@@ -32,6 +32,7 @@ object FieldConverter {
* @param delimiterLevel2 level 2 delimiter for complex type
* @param timeStampFormat timestamp format
* @param dateFormat date format
+ * @param isVarcharType whether it is varchar type. A varchar type has no string length limit
* @param level level for recursive call
*/
def objectToString(
@@ -41,12 +42,14 @@ object FieldConverter {
delimiterLevel2: String,
timeStampFormat: SimpleDateFormat,
dateFormat: SimpleDateFormat,
+ isVarcharType: Boolean = false,
level: Int = 1): String = {
if (value == null) {
serializationNullFormat
} else {
value match {
- case s: String => if (s.length > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
+ case s: String => if (!isVarcharType &&
+ s.length > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
throw new Exception("Dataload failed, String length cannot exceed " +
CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT + " characters")
} else {
@@ -71,7 +74,8 @@ object FieldConverter {
val builder = new StringBuilder()
s.foreach { x =>
builder.append(objectToString(x, serializationNullFormat, delimiterLevel1,
- delimiterLevel2, timeStampFormat, dateFormat, level + 1)).append(delimiter)
+ delimiterLevel2, timeStampFormat, dateFormat, isVarcharType, level + 1))
+ .append(delimiter)
}
builder.substring(0, builder.length - delimiter.length())
case m: scala.collection.Map[Any, Any] =>
@@ -85,7 +89,8 @@ object FieldConverter {
val builder = new StringBuilder()
for (i <- 0 until r.length) {
builder.append(objectToString(r(i), serializationNullFormat, delimiterLevel1,
- delimiterLevel2, timeStampFormat, dateFormat, level + 1)).append(delimiter)
+ delimiterLevel2, timeStampFormat, dateFormat, isVarcharType, level + 1))
+ .append(delimiter)
}
builder.substring(0, builder.length - delimiter.length())
case other => other.toString
[16/50] [abbrv] carbondata git commit: [CARBONDATA-2573] integrate
carbonstore mv branch
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala
index 20b5e8a..241d6a8 100644
--- a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala
@@ -35,7 +35,7 @@ import org.apache.carbondata.mv.plans.util._
*/
package object dsl {
- // object plans {
+ object Plans {
implicit class DslModularPlan(val modularPlan: ModularPlan) {
def select(outputExprs: NamedExpression*)
@@ -96,6 +96,6 @@ package object dsl {
def optimize: LogicalPlan = BirdcageOptimizer.execute(logicalPlan)
}
- // }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala
index 0c5661e..2aff5c0 100644
--- a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala
@@ -30,8 +30,7 @@ object CheckSPJG {
case a: Aggregate =>
a.child.collect {
case Join(_, _, _, _) | Project(_, _) | Filter(_, _) |
-// CatalogRelation(_, _, _) |
- LogicalRelation(_, _, _) | LocalRelation(_, _) => true
+ HiveTableRelation(_, _, _) | LogicalRelation(_, _, _) | LocalRelation(_, _) => true
case _ => false
}.forall(identity)
case _ => false
@@ -59,10 +58,10 @@ object LogicalPlanRule extends SignatureRule[LogicalPlan] {
case LogicalRelation(_, _, _) =>
// TODO: implement this (link to BaseRelation)
None
-// case CatalogRelation(tableMeta, _, _) =>
-// Some(Signature(false,
-// Set(Seq(tableMeta.database, tableMeta.identifier.table).mkString("."))))
- case l: LocalRelation =>
+ case HiveTableRelation(tableMeta, _, _) =>
+ Some(Signature(false,
+ Set(Seq(tableMeta.database, tableMeta.identifier.table).mkString("."))))
+ case l : LocalRelation =>
// LocalRelation is for unit test cases
Some(Signature(groupby = false, Set(l.toString())))
case Filter(_, _) =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala
index 4bc8b97..b6e62eb 100644
--- a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala
@@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicLong
import scala.collection.immutable
-import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, AttributeReference, AttributeSet, Expression, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, AttributeReference, AttributeSet, Cast, Expression, NamedExpression}
import org.apache.spark.sql.catalyst.rules.{Rule, RuleExecutor}
import org.apache.carbondata.mv.expressions.modular._
@@ -71,6 +71,10 @@ class SQLBuilder private(
CleanupQualifier,
// Insert sub queries on top of operators that need to appear after FROM clause.
AddSubquery
+ // Removes [[Cast Casts]] that are unnecessary when converting back to SQL
+ // Comment out for now, will add later by converting AttributMap to Map in SQLBuildDSL
+ // .scala
+ // RemoveCasts
)
)
}
@@ -217,6 +221,14 @@ class SQLBuilder private(
}
}
+ object RemoveCasts extends Rule[ModularPlan] {
+ def apply(tree: ModularPlan): ModularPlan = {
+ tree transformAllExpressions {
+ case Cast(e, dataType, _) => e
+ }
+ }
+ }
+
}
object SQLBuilder {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_Tables.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_Tables.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_Tables.scala
index 97772c7..175b319 100644
--- a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_Tables.scala
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_Tables.scala
@@ -20,7 +20,7 @@ package org.apache.carbondata.mv.testutil
object Tpcds_1_4_Tables {
val tpcds1_4Tables = Seq[String](
s"""
- |CREATE TABLE catalog_sales (
+ |CREATE TABLE IF NOT EXISTS catalog_sales (
| `cs_sold_date_sk` int,
| `cs_sold_time_sk` int,
| `cs_ship_date_sk` int,
@@ -56,10 +56,11 @@ object Tpcds_1_4_Tables {
| `cs_net_paid_inc_ship_tax` decimal(7,2),
| `cs_net_profit` decimal(7,2)
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE catalog_returns (
+ |CREATE TABLE IF NOT EXISTS catalog_returns (
| `cr_returned_date_sk` int,
| `cr_returned_time_sk` int,
| `cr_item_sk` int,
@@ -88,19 +89,21 @@ object Tpcds_1_4_Tables {
| `cr_store_credit` decimal(7,2),
| `cr_net_loss` decimal(7,2)
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE inventory (
+ |CREATE TABLE IF NOT EXISTS inventory (
| `inv_date_sk` int,
| `inv_item_sk` int,
| `inv_warehouse_sk` int,
| `inv_quantity_on_hand` int
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE store_sales (
+ |CREATE TABLE IF NOT EXISTS store_sales (
| `ss_sold_date_sk` int,
| `ss_sold_time_sk` int,
| `ss_item_sk` int,
@@ -125,10 +128,11 @@ object Tpcds_1_4_Tables {
| `ss_net_paid_inc_tax` decimal(7,2),
| `ss_net_profit` decimal(7,2)
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE store_returns (
+ |CREATE TABLE IF NOT EXISTS store_returns (
| `sr_returned_date_sk` int,
| `sr_return_time_sk` int,
| `sr_item_sk` int,
@@ -150,10 +154,11 @@ object Tpcds_1_4_Tables {
| `sr_store_credit` decimal(7,2),
| `sr_net_loss` decimal(7,2)
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE web_sales (
+ |CREATE TABLE IF NOT EXISTS web_sales (
| `ws_sold_date_sk` int,
| `ws_sold_time_sk` int,
| `ws_ship_date_sk` int,
@@ -189,10 +194,11 @@ object Tpcds_1_4_Tables {
| `ws_net_paid_inc_ship_tax` decimal(7,2),
| `ws_net_profit` decimal(7,2)
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE web_returns (
+ |CREATE TABLE IF NOT EXISTS web_returns (
| `wr_returned_date_sk` int,
| `wr_returned_time_sk` int,
| `wr_item_sk` int,
@@ -218,10 +224,11 @@ object Tpcds_1_4_Tables {
| `wr_account_credit` decimal(7,2),
| `wr_net_loss` decimal(7,2)
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE call_center (
+ |CREATE TABLE IF NOT EXISTS call_center (
| `cc_call_center_sk` int,
| `cc_call_center_id` string,
| `cc_rec_start_date` date,
@@ -254,10 +261,11 @@ object Tpcds_1_4_Tables {
| `cc_gmt_offset` decimal(5,2),
| `cc_tax_percentage` decimal(5,2)
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE catalog_page (
+ |CREATE TABLE IF NOT EXISTS catalog_page (
| `cp_catalog_page_sk` int,
| `cp_catalog_page_id` string,
| `cp_start_date_sk` int,
@@ -268,10 +276,11 @@ object Tpcds_1_4_Tables {
| `cp_description` string,
| `cp_type` string
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE customer (
+ |CREATE TABLE IF NOT EXISTS customer (
| `c_customer_sk` int,
| `c_customer_id` string,
| `c_current_cdemo_sk` int,
@@ -291,10 +300,11 @@ object Tpcds_1_4_Tables {
| `c_email_address` string,
| `c_last_review_date` string
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE customer_address (
+ |CREATE TABLE IF NOT EXISTS customer_address (
| `ca_address_sk` int,
| `ca_address_id` string,
| `ca_street_number` string,
@@ -309,10 +319,11 @@ object Tpcds_1_4_Tables {
| `ca_gmt_offset` decimal(5,2),
| `ca_location_type` string
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE customer_demographics (
+ |CREATE TABLE IF NOT EXISTS customer_demographics (
| `cd_demo_sk` int,
| `cd_gender` string,
| `cd_marital_status` string,
@@ -323,10 +334,11 @@ object Tpcds_1_4_Tables {
| `cd_dep_employed_count` int,
| `cd_dep_college_count` int
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE date_dim (
+ |CREATE TABLE IF NOT EXISTS date_dim (
| `d_date_sk` int,
| `d_date_id` string,
| `d_date` date,
@@ -356,28 +368,31 @@ object Tpcds_1_4_Tables {
| `d_current_quarter` string,
| `d_current_year` string
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE household_demographics (
+ |CREATE TABLE IF NOT EXISTS household_demographics (
| `hd_demo_sk` int,
| `hd_income_band_sk` int,
| `hd_buy_potential` string,
| `hd_dep_count` int,
| `hd_vehicle_count` int
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE income_band (
+ |CREATE TABLE IF NOT EXISTS income_band (
| `ib_income_band_sk` int,
| `ib_lower_bound` int,
| `ib_upper_bound` int
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE item (
+ |CREATE TABLE IF NOT EXISTS item (
| `i_item_sk` int,
| `i_item_id` string,
| `i_rec_start_date` date,
@@ -401,10 +416,11 @@ object Tpcds_1_4_Tables {
| `i_manager_id` int,
| `i_product_name` string
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE promotion (
+ |CREATE TABLE IF NOT EXISTS promotion (
| `p_promo_sk` int,
| `p_promo_id` string,
| `p_start_date_sk` int,
@@ -425,18 +441,20 @@ object Tpcds_1_4_Tables {
| `p_purpose` string,
| `p_discount_active` string
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE reason (
+ |CREATE TABLE IF NOT EXISTS reason (
| `r_reason_sk` int,
| `r_reason_id` string,
| `r_reason_desc` string
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE ship_mode (
+ |CREATE TABLE IF NOT EXISTS ship_mode (
| `sm_ship_mode_sk` int,
| `sm_ship_mode_id` string,
| `sm_type` string,
@@ -444,10 +462,11 @@ object Tpcds_1_4_Tables {
| `sm_carrier` string,
| `sm_contract` string
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE store (
+ |CREATE TABLE IF NOT EXISTS store (
| `s_store_sk` int,
| `s_store_id` string,
| `s_rec_start_date` date,
@@ -478,10 +497,11 @@ object Tpcds_1_4_Tables {
| `s_gmt_offset` decimal(5,2),
| `s_tax_precentage` decimal(5,2)
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE time_dim (
+ |CREATE TABLE IF NOT EXISTS time_dim (
| `t_time_sk` int,
| `t_time_id` string,
| `t_time` int,
@@ -493,10 +513,11 @@ object Tpcds_1_4_Tables {
| `t_sub_shift` string,
| `t_meal_time` string
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE warehouse (
+ |CREATE TABLE IF NOT EXISTS warehouse (
| `w_warehouse_sk` int,
| `w_warehouse_id` string,
| `w_warehouse_name` string,
@@ -512,10 +533,11 @@ object Tpcds_1_4_Tables {
| `w_country` string,
| `w_gmt_offset` decimal(5,2)
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE web_page (
+ |CREATE TABLE IF NOT EXISTS web_page (
| `wp_web_page_sk` int,
| `wp_web_page_id` string,
| `wp_rec_start_date` date,
@@ -531,10 +553,11 @@ object Tpcds_1_4_Tables {
| `wp_image_count` int,
| `wp_max_ad_count` int
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE web_site (
+ |CREATE TABLE IF NOT EXISTS web_site (
| `web_site_sk` int,
| `web_site_id` string,
| `web_rec_start_date` date,
@@ -562,10 +585,11 @@ object Tpcds_1_4_Tables {
| `web_gmt_offset` decimal(5,2),
| `web_tax_percentage` decimal(5,2)
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE sdr_dyn_seq_custer_iot_all_hour_60min
+ |CREATE TABLE IF NOT EXISTS sdr_dyn_seq_custer_iot_all_hour_60min
|(
| `dim_1` String,
| `dim_51` String,
@@ -770,10 +794,11 @@ object Tpcds_1_4_Tables {
| `counter_100` double,
| `batchno` double
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE dim_apn_iot
+ |CREATE TABLE IF NOT EXISTS dim_apn_iot
|(
| `city_ascription` String,
| `industry` String,
@@ -782,10 +807,11 @@ object Tpcds_1_4_Tables {
| `customer_name` String,
| `id` bigint
|)
- |STORED BY 'org.apache.carbondata.format'
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+ |STORED AS TEXTFILE
""".stripMargin.trim,
s"""
- |CREATE TABLE tradeflow_all (
+ |CREATE TABLE IF NOT EXISTS tradeflow_all (
| m_month smallint,
| hs_code string ,
| country smallint,
@@ -795,17 +821,17 @@ object Tpcds_1_4_Tables {
| b_country smallint,
| imex smallint,
| y_year smallint)
- |STORED BY 'org.apache.carbondata.format'
+ |STORED AS parquet
""".stripMargin.trim,
s"""
- |CREATE TABLE country (
+ |CREATE TABLE IF NOT EXISTS country (
| countryid smallint ,
| country_en string ,
| country_cn string )
- |STORED BY 'org.apache.carbondata.format'
+ |STORED AS parquet
""".stripMargin.trim,
s"""
- |CREATE TABLE updatetime (
+ |CREATE TABLE IF NOT EXISTS updatetime (
| countryid smallint ,
| imex smallint ,
| hs_len smallint ,
@@ -813,7 +839,7 @@ object Tpcds_1_4_Tables {
| startdate string ,
| newdate string ,
| minnewdate string )
- |STORED BY 'org.apache.carbondata.format'
+ |STORED AS parquet
""".stripMargin.trim
)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/TestSQLBatch.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/TestSQLBatch.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/TestSQLBatch.scala
deleted file mode 100644
index 3806dac..0000000
--- a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/TestSQLBatch.scala
+++ /dev/null
@@ -1,584 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.mv
-
-object TestSQLBatch {
-
- val testSQLBatch = Seq[String](
- s"""
- |SELECT f1.A,COUNT(*) AS B
- |FROM
- | fact f1
- | JOIN dim d1 ON (f1.K = d1.K)
- |WHERE f1.E IS NULL AND (f1.C > d1.E OR d1.E = 3)
- |GROUP BY f1.A
- """.stripMargin.trim,
- s"""
- |SELECT f1.A,COUNT(*) AS B
- |FROM
- | fact f1
- | JOIN dim d1 ON (f1.K = d1.K)
- | JOIN dim1 d2 ON (f1.K = d2.K AND d2.G > 0)
- |WHERE f1.E IS NULL AND f1.C > d1.E
- |GROUP BY f1.A
- """.stripMargin.trim,
- s"""
- |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, date_dim.d_date solddate, count(*) cnt
- |FROM date_dim, store_sales, item
- |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- | AND date_dim.d_year in (2000, 2000+1, 2000+2, 2000+3)
- |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,date_dim.d_date
- """.stripMargin.trim,
- s"""
- |SELECT item.i_item_desc, item.i_category, item.i_class, item.i_current_price,
- | SUM(store_sales.ss_ext_sales_price) as itemrevenue,
- | SUM(store_sales.ss_ext_sales_price)*100/sum(sum(store_sales.ss_ext_sales_price)) over (partition by item.i_class) as revenueratio
- |FROM date_dim, store_sales, item
- |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- | AND item.i_category in ('Sport', 'Books', 'Home')
- | AND date_dim.d_date between cast('1999-02-22' as date) AND (cast('1999-02-22' as date) + interval 30 days)
- |GROUP BY item.i_item_id, item.i_item_desc, item.i_category, item.i_class, item.i_current_price
- """.stripMargin.trim,
- s"""
- |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
- | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
- |FROM date_dim, store_sales, item
- |WHERE store_sales.ss_store_sk IS NULL
- | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- |GROUP BY channel, store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
- """.stripMargin.trim,
- s"""
- |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
- | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
- |FROM date_dim, store_sales, item
- |WHERE store_sales.ss_store_sk IS NULL
- | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- |GROUP BY store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
- """.stripMargin.trim,
- s"""
- |SELECT item.i_brand_id brand_id, item.i_brand brand, SUM(ss_ext_sales_price) ext_price
- |FROM date_dim, store_sales, item
- |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- | AND item.i_manager_id = 28
- | AND date_dim.d_year = 1999
- | AND date_dim.d_moy = 11
- |GROUP BY item.i_brand_id, item.i_brand
- """.stripMargin.trim,
- s"""
- |SELECT item.i_brand_id brand_id, item.i_brand_id brand, SUM(ss_ext_sales_price) ext_price
- |FROM date_dim, store_sales, item
- |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- | AND item.i_manager_id = 28
- | AND date_dim.d_year = 1999
- | AND date_dim.d_moy = 11
- |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
- """.stripMargin.trim,
- s"""
- |SELECT 'store' channel, item.i_brand_id, item.i_class_id, item.i_category_id,
- | SUM(store_sales.ss_quantity*store_sales.ss_list_price) sales, count(*) number_sales
- |FROM date_dim, store_sales, item
- |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- | AND date_dim.d_year = 1999 + 2
- | AND date_dim.d_moy = 11
- |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
- """.stripMargin.trim,
- s"""
- |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, dt.d_date solddate, count(*) cnt
- |FROM date_dim dt, store_sales, item
- |WHERE dt.d_date_sk = store_sales.ss_sold_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- | AND dt.d_year in (2000, 2000+1, 2000+2, 2000+3)
- |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,dt.d_date
- """.stripMargin.trim,
- s"""
- |SELECT fact.B
- |FROM
- | fact
- |UNION ALL
- |SELECT fact.B
- |FROM
- | fact
- |UNION ALL
- |SELECT fact.B
- |FROM
- | fact
- """.stripMargin.trim,
- s"""
- |SELECT fact.A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
- |WHERE fact.C > dim.E AND dim.E IS NULL
- |UNION ALL
- |SELECT fact.A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
- |WHERE fact.C > dim.E AND dim.E IS NULL
- |UNION ALL
- |SELECT fact.B
- |FROM
- | fact
- """.stripMargin.trim,
- s"""
- |SELECT fact.A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
- |WHERE fact.C > dim.E AND dim.E IS NULL
- |UNION ALL
- |SELECT fact.A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
- |WHERE fact.C > dim.E AND dim.E IS NULL
- """.stripMargin.trim,
- s"""
- |SELECT fact.A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
- |WHERE fact.C > dim.E AND dim.E IS NULL
- |UNION ALL
- |SELECT fact.B
- |FROM
- | fact
- """.stripMargin.trim,
- s"""
- |SELECT fact.A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
- |WHERE fact.C > dim.E AND dim.E IS NULL
- |UNION ALL
- |SELECT fact.A
- |FROM
- | fact
- """.stripMargin.trim,
- s"""
- |SELECT f1.A,f1.B,COUNT(*) AS A
- |FROM
- | fact f1
- | JOIN dim d1 ON (f1.K = d1.K)
- |GROUP BY f1.A,f1.B
- """.stripMargin.trim,
- s"""
- |SELECT f1.A,f1.B,COUNT(*) AS A
- |FROM
- | fact f1
- | JOIN dim d1 ON (f1.K = d1.K)
- |WHERE f1.E IS NULL AND f1.C > d1.E AND f1.B = 2
- |GROUP BY f1.A,f1.B
- """.stripMargin.trim,
- s"""
- |SELECT f1.A,f1.B,COUNT(*) AS A
- |FROM
- | fact f1
- | JOIN dim d1 ON (f1.K = d1.K)
- |WHERE f1.E IS NULL AND f1.C > d1.E AND d1.E = 3
- |GROUP BY f1.A,f1.B
- """.stripMargin.trim,
- s"""
- |SELECT f1.A,f1.B,COUNT(*) AS A
- |FROM
- | fact f1
- | JOIN dim d1 ON (f1.K = d1.K)
- |WHERE f1.E IS NULL AND f1.C > d1.E
- |GROUP BY f1.A,f1.B
- """.stripMargin.trim,
- s"""
- |SELECT f1.A,f1.B,COUNT(*) AS A
- |FROM
- | fact f1
- | JOIN dim d1 ON (f1.K = d1.K)
- | JOIN dim d2 ON (f1.K = d2.K AND d2.E > 0)
- |WHERE f1.E IS NULL AND f1.C > d1.E
- |GROUP BY f1.A,f1.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,fact.B,COUNT(*) AS A
- |FROM
- | fact
- | JOIN dim d1 ON (fact.K = d1.K)
- | JOIN dim d2 ON (fact.K = d2.K AND d2.E > 0)
- |WHERE fact.E IS NULL AND fact.C > d1.E
- |GROUP BY fact.A,fact.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K)
- |WHERE fact.C > dim.E AND (dim.E IS NULL OR dim1.G IS NULL)
- """.stripMargin.trim,
- s"""
- |SELECT fact.A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K)
- |WHERE fact.C > dim.E OR dim1.G IS NULL
- """.stripMargin.trim,
- s"""
- |SELECT fact.A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
- |WHERE fact.C > dim.E OR dim.E IS NULL
- """.stripMargin.trim,
- s"""
- |SELECT fact.A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
- |WHERE fact.C > dim.E AND dim.E IS NULL
- """.stripMargin.trim,
- s"""
- |SELECT fact.A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
- |WHERE fact.C > dim.E
- """.stripMargin.trim,
- s"""
- |SELECT fact.A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,fact.B,COUNT(*) AS A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K AND fact.K IS NOT NULL)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0 AND dim1.K IS NOT NULL)
- |WHERE fact.E IS NULL AND fact.C > dim.E
- |GROUP BY fact.A,fact.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,fact.B,COUNT(*) AS A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K AND fact.K IS NOT NULL)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
- |WHERE fact.E IS NULL AND fact.C > dim.E
- |GROUP BY fact.A,fact.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,fact.B,COUNT(*) AS A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
- |WHERE fact.E IS NULL AND fact.C > dim.E
- |GROUP BY fact.A,fact.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,fact.B,COUNT(*) AS A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
- |WHERE fact.C > fact.E AND fact.C > dim.E
- |GROUP BY fact.A,fact.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,fact.B,COUNT(*) AS A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K)
- |WHERE fact.C > fact.E AND (fact.C > dim.E OR dim1.G > 0)
- |GROUP BY fact.A,fact.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,fact.B,COUNT(*) AS A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K)
- |WHERE fact.C > fact.E AND fact.C > dim.E OR dim1.G > 0
- |GROUP BY fact.A,fact.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,fact.B,COUNT(*) AS A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K)
- |WHERE fact.C > fact.E AND fact.C > dim.E
- |GROUP BY fact.A,fact.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,fact.B,COUNT(*) AS A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K)
- |WHERE fact.C > fact.E OR fact.C > dim.E
- |GROUP BY fact.A,fact.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,fact.B,COUNT(*) AS A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K)
- |WHERE fact.C > fact.E
- |GROUP BY fact.A,fact.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,fact.B,COUNT(*) AS A
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K)
- |GROUP BY fact.A,fact.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,COUNT(*) AS S1
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K)
- |GROUP BY fact.A
- |--HAVING COUNT(*) > 5
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,COUNT(*)--, my_fun(3) AS S1
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K)
- |GROUP BY fact.A
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,COUNT(*) AS S1
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K)
- |GROUP BY fact.A
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,sum(cast(dim.D as bigint)) AS S1
- |FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)
- | JOIN dim1 ON (fact.K = dim1.K)
- |GROUP BY fact.A
- """.stripMargin.trim,
- s"""
- |SELECT FOO.A, sum(cast(FOO.B as bigint)) AS S
- |FROM (SELECT fact.A, fact.B
- | FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)) FOO
- |GROUP BY FOO.A
- """.stripMargin.trim,
- s"""
- |SELECT FOO.A, sum(cast(FOO.B as bigint)) AS S
- |FROM (SELECT fact.A, fact.B
- | FROM
- | fact
- | JOIN dim ON (fact.K = dim.K)) FOO
- |GROUP BY FOO.A
- """.stripMargin.trim,
- s"""
- |SELECT f1.A,f1.B,COUNT(*)
- |FROM
- | fact f1
- | JOIN fact f2 ON (f1.K = f2.K)
- | JOIN fact f3 ON (f1.K = f3.K)
- |GROUP BY f1.A,f1.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,fact.B,sum(cast(dim.D as bigint)) AS S1
- |FROM
- | fact
- | LEFT OUTER JOIN dim ON (fact.K = dim.K)
- |GROUP BY fact.A,fact.B
- """.stripMargin.trim,
- s"""
- |SELECT fact.A,fact.B,fact.C,sum(cast(dim.D as bigint)) AS S1
- |FROM
- | fact
- | LEFT OUTER JOIN dim ON (fact.K = dim.K)
- |GROUP BY fact.A,fact.B,fact.C
- """.stripMargin.trim,
-// s"""
-// |SELECT *
-// |FROM fact, dim
-// """.stripMargin.trim,
- s"""
- |SELECT store_sales.ss_store_sk,date_dim.d_year,
- | COUNT(*) numsales
- |FROM date_dim, store_sales
- |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
- |GROUP BY store_sales.ss_store_sk,date_dim.d_year GROUPING SETS (store_sales.ss_store_sk,date_dim.d_year)
- """.stripMargin.trim,
- s"""
- |SELECT store_sales.ss_store_sk,date_dim.d_year,
- | SUM(store_sales.ss_ext_sales_price) as itemrevenue
- |FROM date_dim, store_sales
- |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
- |GROUP BY CUBE(store_sales.ss_store_sk,date_dim.d_year)
- """.stripMargin.trim,
- s"""
- |SELECT date_dim.d_moy,date_dim.d_qoy, date_dim.d_year,
- | SUM(store_sales.ss_ext_sales_price) as itemrevenue
- |FROM date_dim, store_sales
- |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
- |GROUP BY ROLLUP(date_dim.d_moy,date_dim.d_qoy, date_dim.d_year)
- """.stripMargin.trim
- )
- val testSQLBatch2 = Seq[String](
- s"""
- |SELECT f1.A,COUNT(*) AS B
- |FROM
- | fact f1
- | JOIN dim d1 ON (f1.K = d1.K)
- |WHERE f1.E IS NULL AND (f1.C > d1.E OR d1.E = 3)
- |GROUP BY f1.A
- """.stripMargin.trim,
- s"""
- |SELECT f1.A,COUNT(*) AS B
- |FROM
- | fact f1
- | JOIN dim d1 ON (f1.K = d1.K)
- | JOIN dim1 d2 ON (f1.K = d2.K AND d2.G > 0)
- |WHERE f1.E IS NULL AND f1.C > d1.E
- |GROUP BY f1.A
- """.stripMargin.trim,
- s"""
- |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, date_dim.d_date solddate, count(*) cnt
- |FROM date_dim, store_sales, item
- |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- | AND date_dim.d_year in (2000, 2000+1, 2000+2, 2000+3)
- |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,date_dim.d_date
- """.stripMargin.trim,
- s"""
- |SELECT item.i_item_desc, item.i_category, item.i_class, item.i_current_price,
- | SUM(store_sales.ss_ext_sales_price) as itemrevenue,
- | SUM(store_sales.ss_ext_sales_price)*100/sum(sum(store_sales.ss_ext_sales_price)) over (partition by item.i_class) as revenueratio
- |FROM date_dim, store_sales, item
- |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- | AND item.i_category in ('Sport', 'Books', 'Home')
- | AND date_dim.d_date between cast('1999-02-22' as date) AND (cast('1999-02-22' as date) + interval 30 days)
- |GROUP BY item.i_item_id, item.i_item_desc, item.i_category, item.i_class, item.i_current_price
- """.stripMargin.trim,
- s"""
- |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
- | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
- |FROM date_dim, store_sales, item
- |WHERE store_sales.ss_store_sk IS NULL
- | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- |GROUP BY channel, store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
- """.stripMargin.trim,
- s"""
- |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
- | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
- |FROM date_dim, store_sales, item
- |WHERE store_sales.ss_store_sk IS NULL
- | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- |GROUP BY store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
- """.stripMargin.trim,
- s"""
- |SELECT item.i_brand_id brand_id, item.i_brand brand, SUM(ss_ext_sales_price) ext_price
- |FROM date_dim, store_sales, item
- |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- | AND item.i_manager_id = 28
- | AND date_dim.d_year = 1999
- | AND date_dim.d_moy = 11
- |GROUP BY item.i_brand_id, item.i_brand
- """.stripMargin.trim,
- s"""
- |SELECT item.i_brand_id brand_id, item.i_brand_id brand, SUM(ss_ext_sales_price) ext_price
- |FROM date_dim, store_sales, item
- |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- | AND item.i_manager_id = 28
- | AND date_dim.d_year = 1999
- | AND date_dim.d_moy = 11
- |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
- """.stripMargin.trim,
- s"""
- |SELECT 'store' channel, item.i_brand_id, item.i_class_id, item.i_category_id,
- | SUM(store_sales.ss_quantity*store_sales.ss_list_price) sales, count(*) number_sales
- |FROM date_dim, store_sales, item
- |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- | AND date_dim.d_year = 1999 + 2
- | AND date_dim.d_moy = 11
- |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
- """.stripMargin.trim,
- s"""
- |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, dt.d_date solddate, count(*) cnt
- |FROM date_dim dt, store_sales, item
- |WHERE dt.d_date_sk = store_sales.ss_sold_date_sk
- | AND store_sales.ss_item_sk = item.i_item_sk
- | AND dt.d_year in (2000, 2000+1, 2000+2, 2000+3)
- |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,dt.d_date
- """.stripMargin.trim,
-
- s"""
- |SELECT store_sales.ss_store_sk,date_dim.d_year,
- | COUNT(*) numsales
- |FROM date_dim, store_sales
- |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
- |GROUP BY store_sales.ss_store_sk,date_dim.d_year GROUPING SETS (store_sales.ss_store_sk,date_dim.d_year)
- """.stripMargin.trim,
- s"""
- |SELECT store_sales.ss_store_sk,date_dim.d_year,
- | SUM(store_sales.ss_ext_sales_price) as itemrevenue
- |FROM date_dim, store_sales
- |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
- |GROUP BY CUBE(store_sales.ss_store_sk,date_dim.d_year)
- """.stripMargin.trim,
- s"""
- |SELECT date_dim.d_moy,date_dim.d_qoy, date_dim.d_year,
- | SUM(store_sales.ss_ext_sales_price) as itemrevenue
- |FROM date_dim, store_sales
- |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
- |GROUP BY ROLLUP(date_dim.d_moy,date_dim.d_qoy, date_dim.d_year)
- """.stripMargin.trim
- )
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ExtractJoinConditionsSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ExtractJoinConditionsSuite.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ExtractJoinConditionsSuite.scala
index 2e91e80..e1a3d9f 100644
--- a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ExtractJoinConditionsSuite.scala
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ExtractJoinConditionsSuite.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
import org.apache.spark.sql.catalyst.plans.{Inner, _}
-import org.apache.carbondata.mv.dsl._
+import org.apache.carbondata.mv.dsl.Plans._
import org.apache.carbondata.mv.testutil.ModularPlanTest
class ExtractJoinConditionsSuite extends ModularPlanTest {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/IsSPJGHSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/IsSPJGHSuite.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/IsSPJGHSuite.scala
index e80a0cb..dbc1756 100644
--- a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/IsSPJGHSuite.scala
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/IsSPJGHSuite.scala
@@ -20,8 +20,7 @@ package org.apache.carbondata.mv.plans
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
-
-import org.apache.carbondata.mv.dsl._
+import org.apache.carbondata.mv.dsl.Plans._
import org.apache.carbondata.mv.plans.modular.ModularPlan
import org.apache.carbondata.mv.testutil.ModularPlanTest
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/LogicalToModularPlanSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/LogicalToModularPlanSuite.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/LogicalToModularPlanSuite.scala
index 176c5d2..082c325 100644
--- a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/LogicalToModularPlanSuite.scala
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/LogicalToModularPlanSuite.scala
@@ -23,8 +23,7 @@ import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.expressions.aggregate.Count
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.plans.{LeftOuter, RightOuter, _}
-
-import org.apache.carbondata.mv.dsl._
+import org.apache.carbondata.mv.dsl.Plans._
import org.apache.carbondata.mv.plans.modular.Flags._
import org.apache.carbondata.mv.plans.modular.{JoinEdge, ModularRelation}
import org.apache.carbondata.mv.testutil.ModularPlanTest
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ModularToSQLSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ModularToSQLSuite.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ModularToSQLSuite.scala
index 7cd3d73..c74491c 100644
--- a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ModularToSQLSuite.scala
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ModularToSQLSuite.scala
@@ -17,149 +17,133 @@
package org.apache.carbondata.mv.plans
-import org.apache.spark.sql.SparkSession
-import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
+import org.apache.spark.sql.hive.CarbonSessionCatalog
+import org.scalatest.BeforeAndAfter
-import org.apache.carbondata.mv.dsl._
+import org.apache.carbondata.mv.dsl.Plans._
import org.apache.carbondata.mv.testutil.ModularPlanTest
-class ModularToSQLSuite extends ModularPlanTest with BeforeAndAfterAll {
- import org.apache.carbondata.mv.TestSQLBatch._
+class ModularToSQLSuite extends ModularPlanTest with BeforeAndAfter {
- override protected def beforeAll(): Unit = {
- drop
+ import org.apache.carbondata.mv.testutil.TestSQLBatch._
- sql(
- s"""
- |CREATE TABLE Fact (
- | `A` int,
- | `B` int,
- | `C` int,
- | `E` int,
- | `K` int
- |)
- |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
- |STORED AS TEXTFILE
+ val spark = sqlContext
+ val testHive = sqlContext.sparkSession
+ val hiveClient = spark.sparkSession.sessionState.catalog.asInstanceOf[CarbonSessionCatalog].getClient()
+
+ ignore("convert modular plans to sqls") {
+
+ hiveClient.runSqlHive(
+ s"""
+ |CREATE TABLE if not exists Fact (
+ | `A` int,
+ | `B` int,
+ | `C` int,
+ | `E` int,
+ | `K` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim
- )
-
- sql(
- s"""
- |CREATE TABLE Dim (
- | `D` int,
- | `E` int,
- | `K` int
- |)
- |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
- |STORED AS TEXTFILE
+ )
+
+ hiveClient.runSqlHive(
+ s"""
+ |CREATE TABLE if not exists Dim (
+ | `D` int,
+ | `E` int,
+ | `K` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim
- )
-
- sql(
- s"""
- |CREATE TABLE Dim1 (
- | `F` int,
- | `G` int,
- | `K` int
- |)
- |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
- |STORED AS TEXTFILE
+ )
+
+ hiveClient.runSqlHive(
+ s"""
+ |CREATE TABLE if not exists Dim1 (
+ | `F` int,
+ | `G` int,
+ | `K` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim
- )
-
- sql(
- s"""
- |CREATE TABLE store_sales (
- | `ss_sold_date_sk` int,
- | `ss_item_sk` int,
- | `ss_quantity` int,
- | `ss_list_price` decimal(7,2),
- | `ss_ext_sales_price` decimal(7,2),
- | `ss_store_sk` int
- |)
- |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
- |STORED AS TEXTFILE
+ )
+
+ hiveClient.runSqlHive(
+ s"""
+ |CREATE TABLE if not exists store_sales (
+ | `ss_sold_date_sk` int,
+ | `ss_item_sk` int,
+ | `ss_quantity` int,
+ | `ss_list_price` decimal(7,2),
+ | `ss_ext_sales_price` decimal(7,2),
+ | `ss_store_sk` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim
)
-
- sql(
- s"""
- |CREATE TABLE date_dim (
- | `d_date_sk` int,
- | `d_date` date,
- | `d_year` int,
- | `d_moy` int,
- | `d_qoy` int
- |)
- |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
- |STORED AS TEXTFILE
+
+ hiveClient.runSqlHive(
+ s"""
+ |CREATE TABLE if not exists date_dim (
+ | `d_date_sk` int,
+ | `d_date` date,
+ | `d_year` int,
+ | `d_moy` int,
+ | `d_qoy` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim
)
-
- sql(
- s"""
- |CREATE TABLE item (
- | `i_item_sk` int,
- | `i_item_id` string,
- | `i_brand` string,
- | `i_brand_id` int,
- | `i_item_desc` string,
- | `i_class_id` int,
- | `i_class` string,
- | `i_category` string,
- | `i_category_id` int,
- | `i_manager_id` int,
- | `i_current_price` decimal(7,2)
- |)
- |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
- |STORED AS TEXTFILE
+
+ hiveClient.runSqlHive(
+ s"""
+ |CREATE TABLE if not exists item (
+ | `i_item_sk` int,
+ | `i_item_id` string,
+ | `i_brand` string,
+ | `i_brand_id` int,
+ | `i_item_desc` string,
+ | `i_class_id` int,
+ | `i_class` string,
+ | `i_category` string,
+ | `i_category_id` int,
+ | `i_manager_id` int,
+ | `i_current_price` decimal(7,2)
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim
)
-
- sqlContext.udf.register("my_fun", (s: Integer) => s)
- }
-
-
- private def drop = {
- sql(s"drop table if exists Fact")
- sql(s"drop table if exists Dim")
- sql(s"drop table if exists Dim1")
- sql(s"drop table if exists store_sales")
- sql(s"drop table if exists date_dim")
- sql(s"drop table if exists item")
- }
-
- ignore("convert modular plans to sqls") {
+
+ testHive.udf.register("my_fun", (s: Integer) => s)
+
testSQLBatch.foreach { query =>
- testPlan(query)
- }
- }
-
- private def testPlan(query: String) = {
- val analyzed = sql(query).queryExecution.analyzed
- val optimized = analyzed.optimize
- val modularPlan = analyzed.optimize.modularize
+ val analyzed = testHive.sql(query).queryExecution.analyzed
+ val optimized = analyzed.optimize
+ val modularPlan = analyzed.optimize.modularize
- println(s"\n\n===== ACTUAL QUERY =====\n\n${ query } \n")
+ LOGGER.info(s"\n\n===== MODULAR PLAN =====\n\n${modularPlan.treeString} \n")
+
+ val compactSql = modularPlan.asCompactSQL
+ val convertedSql = modularPlan.asOneLineSQL
- println(s"\n\n===== MODULAR PLAN =====\n\n${ modularPlan.treeString } \n")
+ LOGGER.info(s"\n\n===== CONVERTED SQL =====\n\n$compactSql \n")
+
+ val analyzed1 = testHive.sql(convertedSql).queryExecution.analyzed
+ val modularPlan1 = analyzed1.optimize.modularize
- val compactSql = modularPlan.asCompactSQL
- val convertedSql = modularPlan.asOneLineSQL
+ LOGGER.info(s"\n\n===== CONVERTED SQL =====\n\n$compactSql \n")
- println(s"\n\n===== CONVERTED SQL =====\n\n$compactSql \n")
-
- val analyzed1 = sql(convertedSql).queryExecution.analyzed
- val modularPlan1 = analyzed1.optimize.modularize
-
- println(s"\n\n===== CONVERTED SQL =====\n\n$compactSql \n")
-
- println(s"\n\n===== MODULAR PLAN1 =====\n\n${ modularPlan1.treeString } \n")
-
- comparePlans(modularPlan, modularPlan1)
- }
+ LOGGER.info(s"\n\n===== MODULAR PLAN1 =====\n\n${modularPlan1.treeString} \n")
+
+ comparePlans(modularPlan, modularPlan1)
+ }
- override protected def afterAll(): Unit = {
- drop
}
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/SignatureSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/SignatureSuite.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/SignatureSuite.scala
index c64826f..631eca2 100644
--- a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/SignatureSuite.scala
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/SignatureSuite.scala
@@ -18,87 +18,60 @@
package org.apache.carbondata.mv.plans
import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.hive.CarbonSessionCatalog
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.mv.dsl._
+import org.apache.carbondata.mv.dsl.Plans._
import org.apache.carbondata.mv.plans.modular.ModularPlanSignatureGenerator
import org.apache.carbondata.mv.testutil.ModularPlanTest
-import org.apache.carbondata.mv.testutil.Tpcds_1_4_Tables.tpcds1_4Tables
class SignatureSuite extends ModularPlanTest with BeforeAndAfterAll {
- import org.apache.carbondata.mv.TestSQLBatch._
+ import org.apache.carbondata.mv.testutil.TestSQLBatch._
- override protected def beforeAll(): Unit = {
- sql("drop database if exists tpcds1 cascade")
- sql("create database tpcds1")
- sql("use tpcds1")
- tpcds1_4Tables.foreach { create_table =>
- sql(create_table)
- }
+ val spark = sqlContext
+ val testHive = sqlContext.sparkSession
+ val hiveClient = spark.sparkSession.sessionState.catalog.asInstanceOf[CarbonSessionCatalog].getClient()
+
+ ignore("test signature computing") {
- sql(
- s"""
- |CREATE TABLE Fact (
- | `A` int,
- | `B` int,
- | `C` int,
- | `E` int,
- | `K` int
- |)
- |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
- |STORED AS TEXTFILE
+ hiveClient.runSqlHive(
+ s"""
+ |CREATE TABLE if not exists Fact (
+ | `A` int,
+ | `B` int,
+ | `C` int,
+ | `K` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim
- )
-
- sql(
- s"""
- |CREATE TABLE Dim (
- | `D` int,
- | `E` int,
- | `K` int
- |)
- |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
- |STORED AS TEXTFILE
+ )
+
+ hiveClient.runSqlHive(
+ s"""
+ |CREATE TABLE if not exists Dim (
+ | `D` int,
+ | `K` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
""".stripMargin.trim
- )
-
- sql(
- s"""
- |CREATE TABLE Dim1 (
- | `F` int,
- | `G` int,
- | `K` int
- |)
- |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
- |STORED AS TEXTFILE
- """.stripMargin.trim
- )
-
- sqlContext.udf.register("my_fun", (s: Integer) => s)
- }
-
-
- test("test signature computing") {
-
+ )
+
testSQLBatch.foreach { query =>
- val analyzed = sql(query).queryExecution.analyzed
+ val analyzed = testHive.sql(query).queryExecution.analyzed
val modularPlan = analyzed.optimize.modularize
val sig = ModularPlanSignatureGenerator.generate(modularPlan)
sig match {
case Some(s) if (s.groupby != true || s.datasets != Set("default.fact","default.dim")) =>
- println(
+ fail(
s"""
|=== FAIL: signature do not match ===
|${sideBySide(s.groupby.toString, true.toString).mkString("\n")}
|${sideBySide(s.datasets.toString, Set("Fact","Dim").toString).mkString("\n")}
""".stripMargin)
case _ =>
- }
+ }
}
- }
-
- override protected def afterAll(): Unit = {
- sql("use default")
- sql("drop database if exists tpcds1 cascade")
- }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/Tpcds_1_4_BenchmarkSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/Tpcds_1_4_BenchmarkSuite.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/Tpcds_1_4_BenchmarkSuite.scala
new file mode 100644
index 0000000..f8441f9
--- /dev/null
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/Tpcds_1_4_BenchmarkSuite.scala
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans
+
+import scala.util.{Failure, Success, Try}
+import org.apache.spark.sql.SparkSession
+import org.scalatest.BeforeAndAfter
+import org.apache.carbondata.mv.dsl._
+import org.apache.carbondata.mv.testutil.ModularPlanTest
+
+// scalastyle:off println
+class Tpcds_1_4_BenchmarkSuite extends ModularPlanTest with BeforeAndAfter {
+ import org.apache.carbondata.mv.testutil.Tpcds_1_4_QueryBatch._
+ import org.apache.carbondata.mv.testutil.Tpcds_1_4_Tables._
+
+// val spark = SparkSession.builder().master("local").enableHiveSupport().getOrCreate()
+// // spark.conf.set("spark.sql.crossJoin.enabled", true)
+// val testHive = new org.apache.spark.sql.hive.test.TestHiveContext(spark.sparkContext, false)
+// val hiveClient = testHive.sparkSession.metadataHive
+
+// test("test SQLBuilder using tpc-ds queries") {
+//
+// tpcds1_4Tables.foreach { create_table =>
+// hiveClient.runSqlHive(create_table)
+// }
+//
+//// val dest = "qTradeflow" // this line is for development, comment it out once done
+// val dest = "qSEQ"
+//// val dest = "qAggPushDown" // this line is for development, comment it out once done
+//// val dest = "q10"
+//
+// tpcds1_4Queries.foreach { query =>
+// if (query._1 == dest) { // this line is for development, comment it out once done
+// val analyzed = testHive.sql(query._2).queryExecution.analyzed
+// println(s"""\n\n===== Analyzed Logical Plan for ${query._1} =====\n\n$analyzed \n""")
+//
+//// val cnonicalizedPlan = new SQLBuilder(analyzed).Canonicalizer.execute(analyzed)
+////
+//// Try(new SQLBuilder(analyzed).toSQL) match {
+//// case Success(s) => logInfo(s"""\n\n===== CONVERTED back ${query._1} USING SQLBuilder =====\n\n$s \n""")
+//// case Failure(e) => logInfo(s"""Cannot convert the logical query plan of ${query._1} back to SQL""")
+//// }
+//
+// // this Try is for development, comment it out once done
+// Try(analyzed.optimize) match {
+// case Success(o) => {
+// println(s"""\n\n===== Optimized Logical Plan for ${query._1} =====\n\n$o \n""")
+// }
+// case Failure(e) =>
+// }
+//
+// val o = analyzed.optimize
+// val o1 = o.modularize
+//
+// Try(o.modularize.harmonize) match {
+// case Success(m) => {
+// println(s"""\n\n===== MODULAR PLAN for ${query._1} =====\n\n$m \n""")
+//
+// Try(m.asCompactSQL) match {
+// case Success(s) => println(s"\n\n===== CONVERTED SQL for ${query._1} =====\n\n${s}\n")
+// case Failure(e) => println(s"""\n\n===== CONVERTED SQL for ${query._1} failed =====\n\n${e.toString}""")
+// }
+// }
+// case Failure(e) => println(s"""\n\n==== MODULARIZE the logical query plan for ${query._1} failed =====\n\n${e.toString}""")
+// }
+// }
+// }
+//
+// }
+}
+// scalastyle:on println
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/testutil/TestSQLBatch.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/testutil/TestSQLBatch.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/testutil/TestSQLBatch.scala
new file mode 100644
index 0000000..bb90340
--- /dev/null
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/testutil/TestSQLBatch.scala
@@ -0,0 +1,584 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.testutil
+
+object TestSQLBatch {
+
+ val testSQLBatch = Seq[String](
+ s"""
+ |SELECT f1.A,COUNT(*) AS B
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ |WHERE f1.E IS NULL AND (f1.C > d1.E OR d1.E = 3)
+ |GROUP BY f1.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,COUNT(*) AS B
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ | JOIN dim1 d2 ON (f1.K = d2.K AND d2.G > 0)
+ |WHERE f1.E IS NULL AND f1.C > d1.E
+ |GROUP BY f1.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, date_dim.d_date solddate, count(*) cnt
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND date_dim.d_year in (2000, 2000+1, 2000+2, 2000+3)
+ |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,date_dim.d_date
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_item_desc, item.i_category, item.i_class, item.i_current_price,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue,
+ | SUM(store_sales.ss_ext_sales_price)*100/sum(sum(store_sales.ss_ext_sales_price)) over (partition by item.i_class) as revenueratio
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_category in ('Sport', 'Books', 'Home')
+ | AND date_dim.d_date between cast('1999-02-22' as date) AND (cast('1999-02-22' as date) + interval 30 days)
+ |GROUP BY item.i_item_id, item.i_item_desc, item.i_category, item.i_class, item.i_current_price
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
+ | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_store_sk IS NULL
+ | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ |GROUP BY channel, store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
+ | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_store_sk IS NULL
+ | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ |GROUP BY store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_brand_id brand_id, item.i_brand brand, SUM(ss_ext_sales_price) ext_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_manager_id = 28
+ | AND date_dim.d_year = 1999
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_brand
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_brand_id brand_id, item.i_brand_id brand, SUM(ss_ext_sales_price) ext_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_manager_id = 28
+ | AND date_dim.d_year = 1999
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, item.i_brand_id, item.i_class_id, item.i_category_id,
+ | SUM(store_sales.ss_quantity*store_sales.ss_list_price) sales, count(*) number_sales
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND date_dim.d_year = 1999 + 2
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
+ """.stripMargin.trim,
+ s"""
+ |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, dt.d_date solddate, count(*) cnt
+ |FROM date_dim dt, store_sales, item
+ |WHERE dt.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND dt.d_year in (2000, 2000+1, 2000+2, 2000+3)
+ |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,dt.d_date
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.B
+ |FROM
+ | fact
+ |UNION ALL
+ |SELECT fact.B
+ |FROM
+ | fact
+ |UNION ALL
+ |SELECT fact.B
+ |FROM
+ | fact
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ |UNION ALL
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ |UNION ALL
+ |SELECT fact.B
+ |FROM
+ | fact
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ |UNION ALL
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ |UNION ALL
+ |SELECT fact.B
+ |FROM
+ | fact
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ |UNION ALL
+ |SELECT fact.A
+ |FROM
+ | fact
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,f1.B,COUNT(*) AS A
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ |GROUP BY f1.A,f1.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,f1.B,COUNT(*) AS A
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ |WHERE f1.E IS NULL AND f1.C > d1.E AND f1.B = 2
+ |GROUP BY f1.A,f1.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,f1.B,COUNT(*) AS A
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ |WHERE f1.E IS NULL AND f1.C > d1.E AND d1.E = 3
+ |GROUP BY f1.A,f1.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,f1.B,COUNT(*) AS A
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ |WHERE f1.E IS NULL AND f1.C > d1.E
+ |GROUP BY f1.A,f1.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,f1.B,COUNT(*) AS A
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ | JOIN dim d2 ON (f1.K = d2.K AND d2.E > 0)
+ |WHERE f1.E IS NULL AND f1.C > d1.E
+ |GROUP BY f1.A,f1.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim d1 ON (fact.K = d1.K)
+ | JOIN dim d2 ON (fact.K = d2.K AND d2.E > 0)
+ |WHERE fact.E IS NULL AND fact.C > d1.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > dim.E AND (dim.E IS NULL OR dim1.G IS NULL)
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > dim.E OR dim1.G IS NULL
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E OR dim.E IS NULL
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K AND fact.K IS NOT NULL)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0 AND dim1.K IS NOT NULL)
+ |WHERE fact.E IS NULL AND fact.C > dim.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K AND fact.K IS NOT NULL)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.E IS NULL AND fact.C > dim.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.E IS NULL AND fact.C > dim.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > fact.E AND fact.C > dim.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > fact.E AND (fact.C > dim.E OR dim1.G > 0)
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > fact.E AND fact.C > dim.E OR dim1.G > 0
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > fact.E AND fact.C > dim.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > fact.E OR fact.C > dim.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > fact.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,COUNT(*) AS S1
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |GROUP BY fact.A
+ |--HAVING COUNT(*) > 5
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,COUNT(*)--, my_fun(3) AS S1
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |GROUP BY fact.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,COUNT(*) AS S1
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |GROUP BY fact.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,sum(cast(dim.D as bigint)) AS S1
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |GROUP BY fact.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT FOO.A, sum(cast(FOO.B as bigint)) AS S
+ |FROM (SELECT fact.A, fact.B
+ | FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)) FOO
+ |GROUP BY FOO.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT FOO.A, sum(cast(FOO.B as bigint)) AS S
+ |FROM (SELECT fact.A, fact.B
+ | FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)) FOO
+ |GROUP BY FOO.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,f1.B,COUNT(*)
+ |FROM
+ | fact f1
+ | JOIN fact f2 ON (f1.K = f2.K)
+ | JOIN fact f3 ON (f1.K = f3.K)
+ |GROUP BY f1.A,f1.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,sum(cast(dim.D as bigint)) AS S1
+ |FROM
+ | fact
+ | LEFT OUTER JOIN dim ON (fact.K = dim.K)
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,fact.C,sum(cast(dim.D as bigint)) AS S1
+ |FROM
+ | fact
+ | LEFT OUTER JOIN dim ON (fact.K = dim.K)
+ |GROUP BY fact.A,fact.B,fact.C
+ """.stripMargin.trim,
+// s"""
+// |SELECT *
+// |FROM fact, dim
+// """.stripMargin.trim,
+ s"""
+ |SELECT store_sales.ss_store_sk,date_dim.d_year,
+ | COUNT(*) numsales
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY store_sales.ss_store_sk,date_dim.d_year GROUPING SETS (store_sales.ss_store_sk,date_dim.d_year)
+ """.stripMargin.trim,
+ s"""
+ |SELECT store_sales.ss_store_sk,date_dim.d_year,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY CUBE(store_sales.ss_store_sk,date_dim.d_year)
+ """.stripMargin.trim,
+ s"""
+ |SELECT date_dim.d_moy,date_dim.d_qoy, date_dim.d_year,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY ROLLUP(date_dim.d_moy,date_dim.d_qoy, date_dim.d_year)
+ """.stripMargin.trim
+ )
+ val testSQLBatch2 = Seq[String](
+ s"""
+ |SELECT f1.A,COUNT(*) AS B
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ |WHERE f1.E IS NULL AND (f1.C > d1.E OR d1.E = 3)
+ |GROUP BY f1.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,COUNT(*) AS B
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ | JOIN dim1 d2 ON (f1.K = d2.K AND d2.G > 0)
+ |WHERE f1.E IS NULL AND f1.C > d1.E
+ |GROUP BY f1.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, date_dim.d_date solddate, count(*) cnt
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND date_dim.d_year in (2000, 2000+1, 2000+2, 2000+3)
+ |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,date_dim.d_date
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_item_desc, item.i_category, item.i_class, item.i_current_price,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue,
+ | SUM(store_sales.ss_ext_sales_price)*100/sum(sum(store_sales.ss_ext_sales_price)) over (partition by item.i_class) as revenueratio
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_category in ('Sport', 'Books', 'Home')
+ | AND date_dim.d_date between cast('1999-02-22' as date) AND (cast('1999-02-22' as date) + interval 30 days)
+ |GROUP BY item.i_item_id, item.i_item_desc, item.i_category, item.i_class, item.i_current_price
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
+ | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_store_sk IS NULL
+ | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ |GROUP BY channel, store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
+ | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_store_sk IS NULL
+ | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ |GROUP BY store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_brand_id brand_id, item.i_brand brand, SUM(ss_ext_sales_price) ext_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_manager_id = 28
+ | AND date_dim.d_year = 1999
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_brand
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_brand_id brand_id, item.i_brand_id brand, SUM(ss_ext_sales_price) ext_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_manager_id = 28
+ | AND date_dim.d_year = 1999
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, item.i_brand_id, item.i_class_id, item.i_category_id,
+ | SUM(store_sales.ss_quantity*store_sales.ss_list_price) sales, count(*) number_sales
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND date_dim.d_year = 1999 + 2
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
+ """.stripMargin.trim,
+ s"""
+ |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, dt.d_date solddate, count(*) cnt
+ |FROM date_dim dt, store_sales, item
+ |WHERE dt.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND dt.d_year in (2000, 2000+1, 2000+2, 2000+3)
+ |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,dt.d_date
+ """.stripMargin.trim,
+
+ s"""
+ |SELECT store_sales.ss_store_sk,date_dim.d_year,
+ | COUNT(*) numsales
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY store_sales.ss_store_sk,date_dim.d_year GROUPING SETS (store_sales.ss_store_sk,date_dim.d_year)
+ """.stripMargin.trim,
+ s"""
+ |SELECT store_sales.ss_store_sk,date_dim.d_year,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY CUBE(store_sales.ss_store_sk,date_dim.d_year)
+ """.stripMargin.trim,
+ s"""
+ |SELECT date_dim.d_moy,date_dim.d_qoy, date_dim.d_year,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY ROLLUP(date_dim.d_moy,date_dim.d_qoy, date_dim.d_year)
+ """.stripMargin.trim
+ )
+}
\ No newline at end of file
[15/50] [abbrv] carbondata git commit: [CARBONDATA-2573] integrate
carbonstore mv branch
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/testutil/TestSQLBatch2.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/testutil/TestSQLBatch2.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/testutil/TestSQLBatch2.scala
new file mode 100644
index 0000000..a02cc89
--- /dev/null
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/testutil/TestSQLBatch2.scala
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.testutil
+
+object TestSQLBatch2 {
+
+ val testSQLBatch2 = Seq[String](
+ s"""
+ |SELECT f1.A,COUNT(*) AS B
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ |WHERE f1.E IS NULL AND (f1.C > d1.E OR d1.E = 3)
+ |GROUP BY f1.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,COUNT(*) AS B
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ | JOIN dim1 d2 ON (f1.K = d2.K AND d2.G > 0)
+ |WHERE f1.E IS NULL AND f1.C > d1.E
+ |GROUP BY f1.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, date_dim.d_date solddate, count(*) cnt
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND date_dim.d_year in (2000, 2000+1, 2000+2, 2000+3)
+ |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,date_dim.d_date
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_item_desc, item.i_category, item.i_class, item.i_current_price,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue,
+ | SUM(store_sales.ss_ext_sales_price)*100/sum(sum(store_sales.ss_ext_sales_price)) over (partition by item.i_class) as revenueratio
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_category in ('Sport', 'Books', 'Home')
+ | AND date_dim.d_date between cast('1999-02-22' as date) AND (cast('1999-02-22' as date) + interval 30 days)
+ |GROUP BY item.i_item_id, item.i_item_desc, item.i_category, item.i_class, item.i_current_price
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
+ | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_store_sk IS NULL
+ | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ |GROUP BY channel, store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
+ | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_store_sk IS NULL
+ | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ |GROUP BY store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_brand_id brand_id, item.i_brand brand, SUM(ss_ext_sales_price) ext_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_manager_id = 28
+ | AND date_dim.d_year = 1999
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_brand
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_brand_id brand_id, item.i_brand_id brand, SUM(ss_ext_sales_price) ext_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_manager_id = 28
+ | AND date_dim.d_year = 1999
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, item.i_brand_id, item.i_class_id, item.i_category_id,
+ | SUM(store_sales.ss_quantity*store_sales.ss_list_price) sales, count(*) number_sales
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND date_dim.d_year = 1999 + 2
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
+ """.stripMargin.trim,
+ s"""
+ |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, dt.d_date solddate, count(*) cnt
+ |FROM date_dim dt, store_sales, item
+ |WHERE dt.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND dt.d_year in (2000, 2000+1, 2000+2, 2000+3)
+ |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,dt.d_date
+ """.stripMargin.trim,
+
+ s"""
+ |SELECT store_sales.ss_store_sk,date_dim.d_year,
+ | COUNT(*) numsales
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY store_sales.ss_store_sk,date_dim.d_year GROUPING SETS (store_sales.ss_store_sk,date_dim.d_year)
+ """.stripMargin.trim,
+ s"""
+ |SELECT store_sales.ss_store_sk,date_dim.d_year,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY CUBE(store_sales.ss_store_sk,date_dim.d_year)
+ """.stripMargin.trim,
+ s"""
+ |SELECT date_dim.d_moy,date_dim.d_qoy, date_dim.d_year,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY ROLLUP(date_dim.d_moy,date_dim.d_qoy, date_dim.d_year)
+ """.stripMargin.trim
+ )
+}
\ No newline at end of file
[33/50] [abbrv] carbondata git commit: [CARBONDATA-2610] Fix for
datamap creation failed on table having loaded data with null value on string
datatype
Posted by ja...@apache.org.
[CARBONDATA-2610] Fix for datamap creation failed on table having loaded data with null value on string datatype
Problem: Datamap creation having null values already loaded in string datatype of table fails.
Solution: Check for null before converting data to the string.
This closes #2376
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e7fed361
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e7fed361
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e7fed361
Branch: refs/heads/carbonstore
Commit: e7fed361b93d7986392c469009f0f6633d71def5
Parents: ece0672
Author: Jatin <ja...@knoldus.in>
Authored: Thu Jun 14 22:56:09 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Tue Jun 19 00:23:01 2018 +0800
----------------------------------------------------------------------
.../lucene/LuceneFineGrainDataMapSuite.scala | 31 ++++++++++++++++++++
.../datamap/IndexDataMapRebuildRDD.scala | 2 +-
2 files changed, 32 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7fed361/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
index 6530ec0..6d774a2 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -829,6 +829,37 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
sql("select * from table_stop where text_match('suggestion:*is*')").collect().length == 1)
}
+ test("test lucene data map on null values") {
+ sql("DROP TABLE IF EXISTS datamap_test4")
+ sql("DROP TABLE IF EXISTS datamap_copy")
+ sql(
+ """
+ | CREATE TABLE datamap_test4(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT', 'autorefreshdatamap' = 'false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE datamap_copy(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT', 'autorefreshdatamap' = 'false')
+ """.stripMargin)
+ sql("insert into datamap_test4 select 1,'name','city',20")
+ sql("insert into datamap_test4 select 2,'name1','city1',20")
+ sql("insert into datamap_test4 select 25,cast(null as string),'city2',NULL")
+ sql("insert into datamap_copy select * from datamap_test4")
+ sql(
+ s"""
+ | CREATE DATAMAP dm4 ON TABLE datamap_test4
+ | USING 'lucene'
+ | DMProperties('INDEX_COLUMNS'='name , city')
+ """.stripMargin)
+ checkAnswer(sql("SELECT * FROM datamap_test4 WHERE TEXT_MATCH('name:n*')"),
+ sql(s"select * from datamap_copy where name like '%n%'"))
+ sql("drop table datamap_test4")
+ sql("drop table datamap_copy")
+ }
+
override protected def afterAll(): Unit = {
LuceneFineGrainDataMapSuite.deleteFile(file2)
sql("DROP TABLE IF EXISTS normal_test")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7fed361/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
index f3f2650..cde6201 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
@@ -124,7 +124,7 @@ class OriginalReadSupport(dataTypes: Array[DataType]) extends CarbonReadSupport[
override def readRow(data: Array[Object]): Array[Object] = {
dataTypes.zipWithIndex.foreach { case (dataType, i) =>
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.STRING && data(i) != null) {
data(i) = data(i).toString
}
}
[44/50] [abbrv] carbondata git commit:
[CARBONDATA-2616][BloomDataMap] Fix bugs in querying bloom datamap with two
index columns
Posted by ja...@apache.org.
[CARBONDATA-2616][BloomDataMap] Fix bugs in querying bloom datamap with two index columns
During pruning in bloomfilter datamap, the same blocklets has been added
to result more than once, thus causing explaining and querying returning
incorrect result.
This closes #2386
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6eb360e1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6eb360e1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6eb360e1
Branch: refs/heads/carbonstore
Commit: 6eb360e1f5f577a576d185efb7dcbf1cc6a302e8
Parents: 01b48fc
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Wed Jun 20 16:31:28 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Jun 21 12:03:48 2018 +0800
----------------------------------------------------------------------
.../datamap/bloom/BloomCoarseGrainDataMap.java | 5 +++--
.../bloom/BloomCoarseGrainDataMapSuite.scala | 18 +++++++++++++++++-
2 files changed, 20 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6eb360e1/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
index e9af0ff..ed03256 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -85,7 +86,7 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
@Override
public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
List<PartitionSpec> partitions) {
- List<Blocklet> hitBlocklets = new ArrayList<Blocklet>();
+ Set<Blocklet> hitBlocklets = new HashSet<>();
if (filterExp == null) {
// null is different from empty here. Empty means after pruning, no blocklet need to scan.
return null;
@@ -111,7 +112,7 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
}
}
}
- return hitBlocklets;
+ return new ArrayList<>(hitBlocklets);
}
private byte[] convertValueToBytes(DataType dataType, Object value) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6eb360e1/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
index 7df3901..c9a4097 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
@@ -71,7 +71,23 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with
checkAnswer(
checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city = 'city_999'", dataMapName, shouldHit),
sql(s"select * from $normalTable where city = 'city_999'"))
- checkAnswer(
+ // query with two index_columns
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id = 1 and city='city_1'", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where id = 1 and city='city_1'"))
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id = 999 and city='city_999'", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where id = 999 and city='city_999'"))
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city = 'city_1' and id = 0", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where city = 'city_1' and id = 0"))
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city = 'city_999' and name='n999'", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where city = 'city_999' and name='n999'"))
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city = 'city_999' and name='n1'", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where city = 'city_999' and name='n1'"))
+ checkAnswer(
sql(s"select min(id), max(id), min(name), max(name), min(city), max(city)" +
s" from $bloomDMSampleTable"),
sql(s"select min(id), max(id), min(name), max(name), min(city), max(city)" +
[08/50] [abbrv] carbondata git commit: [CARONDATA-2559]task id set
for each carbonReader in threadlocal
Posted by ja...@apache.org.
[CARONDATA-2559]task id set for each carbonReader in threadlocal
1. Task Id set for CarbonReader because for each CarbonReader object it should be separate Thread Local variable .
2. If sort-Column is not given to CarbonWriter Describe formatted showing default sort_cols is fixed
3. Issue : CarbonReader was being closed after one iteration. So when reader iterates over the next batch it gives NullPointerException because it is already closed.
Solution : reader is closed if any exception encountered. Else It will be closed explicitly by user.
4. CarbonProperties API for SDK moved to common API List because Property setting is common for both(carbonReader and CarbonWriter) .
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/92d9b925
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/92d9b925
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/92d9b925
Branch: refs/heads/carbonstore
Commit: 92d9b9256373763f05736e29d93b7e835e0da3dd
Parents: 4bb7e27
Author: rahulforallp <ra...@knoldus.in>
Authored: Tue May 29 10:23:46 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Mon Jun 4 17:49:05 2018 +0530
----------------------------------------------------------------------
docs/sdk-guide.md | 95 ++++++++++----------
.../TestNonTransactionalCarbonTable.scala | 13 +--
.../carbondata/sdk/file/CarbonReader.java | 5 ++
.../sdk/file/CarbonReaderBuilder.java | 10 ++-
.../sdk/file/CarbonWriterBuilder.java | 4 +-
.../sdk/file/CSVCarbonWriterTest.java | 2 +-
.../carbondata/sdk/file/CarbonReaderTest.java | 41 ++++-----
.../apache/carbondata/sdk/file/TestUtil.java | 4 +-
.../carbondata/store/LocalCarbonStoreTest.java | 2 +-
9 files changed, 96 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92d9b925/docs/sdk-guide.md
----------------------------------------------------------------------
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index 2371b33..5dbb5ac 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -350,52 +350,6 @@ public Schema(Field[] fields);
public static Schema parseJson(String json);
```
-### Class org.apache.carbondata.core.util.CarbonProperties
-
-```
-/**
-* This method will be responsible to get the instance of CarbonProperties class
-*
-* @return carbon properties instance
-*/
-public static CarbonProperties getInstance();
-```
-
-```
-/**
-* This method will be used to add a new property
-*
-* @param key is a property name to set for carbon.
-* @param value is valid parameter corresponding to property.
-* @return CarbonProperties object
-*/
-public CarbonProperties addProperty(String key, String value);
-```
-
-```
-/**
-* This method will be used to get the property value. If property is not
-* present, then it will return the default value.
-*
-* @param key is a property name to get user specified value.
-* @return properties value for corresponding key. If not set, then returns null.
-*/
-public String getProperty(String key);
-```
-
-```
-/**
-* This method will be used to get the property value. If property is not
-* present, then it will return the default value.
-*
-* @param key is a property name to get user specified value..
-* @param defaultValue used to be returned by function if corrosponding key not set.
-* @return properties value for corresponding key. If not set, then returns specified defaultValue.
-*/
-public String getProperty(String key, String defaultValue);
-```
-Reference : [list of carbon properties](http://carbondata.apache.org/configuration-parameters.html)
-
### Class org.apache.carbondata.sdk.file.AvroCarbonWriter
```
/**
@@ -705,3 +659,52 @@ Find example code at [CarbonReaderExample](https://github.com/apache/carbondata/
```
Find S3 example code at [SDKS3Example](https://github.com/apache/carbondata/blob/master/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java) in the CarbonData repo.
+
+
+# Common API List for CarbonReader and CarbonWriter
+
+### Class org.apache.carbondata.core.util.CarbonProperties
+
+```
+/**
+* This method will be responsible to get the instance of CarbonProperties class
+*
+* @return carbon properties instance
+*/
+public static CarbonProperties getInstance();
+```
+
+```
+/**
+* This method will be used to add a new property
+*
+* @param key is a property name to set for carbon.
+* @param value is valid parameter corresponding to property.
+* @return CarbonProperties object
+*/
+public CarbonProperties addProperty(String key, String value);
+```
+
+```
+/**
+* This method will be used to get the property value. If property is not
+* present, then it will return the default value.
+*
+* @param key is a property name to get user specified value.
+* @return properties value for corresponding key. If not set, then returns null.
+*/
+public String getProperty(String key);
+```
+
+```
+/**
+* This method will be used to get the property value. If property is not
+* present, then it will return the default value.
+*
+* @param key is a property name to get user specified value..
+* @param defaultValue used to be returned by function if corrosponding key not set.
+* @return properties value for corresponding key. If not set, then returns specified defaultValue.
+*/
+public String getProperty(String key, String defaultValue);
+```
+Reference : [list of carbon properties](http://carbondata.apache.org/configuration-parameters.html)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92d9b925/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index 0083733..5beb9c4 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -378,7 +378,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
- checkExistence(sql("describe formatted sdkOutputTable"), true, "name")
+ checkExistence(sql("describe formatted sdkOutputTable"), true, "SORT_COLUMNS name")
buildTestDataWithSortColumns(List())
assert(new File(writerPath).exists())
@@ -390,15 +390,18 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
|'carbondata' LOCATION
|'$writerPath' """.stripMargin)
- sql("describe formatted sdkOutputTable").show(false)
+ checkExistence(sql("describe formatted sdkOutputTable"),false,"SORT_COLUMNS name")
sql("select * from sdkOutputTable").show()
+ sql("DROP TABLE sdkOutputTable")
+ // drop table should not delete the files
+ assert(new File(writerPath).exists())
+ cleanTestData()
+
intercept[RuntimeException] {
buildTestDataWithSortColumns(List(""))
}
-
- sql("DROP TABLE sdkOutputTable")
- // drop table should not delete the files
+
assert(!(new File(writerPath).exists()))
cleanTestData()
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92d9b925/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
index 81db7b2..9af710f 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
@@ -24,6 +24,8 @@ import java.util.List;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.util.CarbonTaskInfo;
+import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
import org.apache.hadoop.mapreduce.RecordReader;
@@ -54,6 +56,9 @@ public class CarbonReader<T> {
this.readers = readers;
this.index = 0;
this.currentReader = readers.get(0);
+ CarbonTaskInfo carbonTaskInfo = new CarbonTaskInfo();
+ carbonTaskInfo.setTaskId(System.nanoTime());
+ ThreadLocalTaskInfo.setCarbonTaskInfo(carbonTaskInfo);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92d9b925/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
index e99ff0d..9d7470e 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
@@ -233,9 +233,13 @@ public class CarbonReaderBuilder {
TaskAttemptContextImpl attempt =
new TaskAttemptContextImpl(job.getConfiguration(), new TaskAttemptID());
RecordReader reader = format.createRecordReader(split, attempt);
- reader.initialize(split, attempt);
- reader.close();
- readers.add(reader);
+ try {
+ reader.initialize(split, attempt);
+ readers.add(reader);
+ } catch (Exception e) {
+ reader.close();
+ throw e;
+ }
}
return new CarbonReader<>(readers);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92d9b925/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index e2dc8c2..bd64568 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -379,7 +379,7 @@ public class CarbonWriterBuilder {
}
List<String> sortColumnsList = new ArrayList<>();
- if (sortColumns == null || sortColumns.length == 0) {
+ if (sortColumns == null) {
// If sort columns are not specified, default set all dimensions to sort column.
// When dimensions are default set to sort column,
// Inverted index will be supported by default for sort columns.
@@ -484,7 +484,7 @@ public class CarbonWriterBuilder {
if (isSortColumn > -1) {
columnSchema.setSortColumn(true);
sortColumnsSchemaList[isSortColumn] = columnSchema;
- } else if (sortColumnsList.isEmpty() && columnSchema.isDimensionColumn()
+ } else if (!sortColumnsList.isEmpty() && columnSchema.isDimensionColumn()
&& columnSchema.getNumberOfChild() < 1) {
columnSchema.setSortColumn(true);
sortColumnsSchemaList[i] = columnSchema;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92d9b925/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
index 1eed47b..865097b 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
@@ -205,7 +205,7 @@ public class CSVCarbonWriterTest {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
+ TestUtil.writeFilesAndVerify(100, new Schema(fields), path, true);
String schemaFile = CarbonTablePath.getSchemaFilePath(path);
Assert.assertTrue(new File(schemaFile).exists());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92d9b925/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index 95c25f8..db118cd 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -59,28 +59,28 @@ public class CarbonReaderTest extends TestCase {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path, true);
CarbonReader reader = CarbonReader.builder(path, "_temp").isTransactionalTable(true)
.projection(new String[]{"name", "age"}).build();
// expected output after sorting
- String[] name = new String[100];
- int[] age = new int[100];
- for (int i = 0; i < 100; i++) {
+ String[] name = new String[200];
+ Integer[] age = new Integer[200];
+ for (int i = 0; i < 200; i++) {
name[i] = "robot" + (i / 10);
- age[i] = (i % 10) * 10 + i / 10;
+ age[i] = i;
}
int i = 0;
while (reader.hasNext()) {
Object[] row = (Object[]) reader.readNextRow();
// Default sort column is applied for dimensions. So, need to validate accordingly
- Assert.assertEquals(name[i], row[0]);
- Assert.assertEquals(age[i], row[1]);
+ assert(Arrays.asList(name).contains(row[0]));
+ assert(Arrays.asList(age).contains(row[1]));
i++;
}
- Assert.assertEquals(i, 100);
+ Assert.assertEquals(i, 200);
reader.close();
@@ -95,11 +95,11 @@ public class CarbonReaderTest extends TestCase {
while (reader2.hasNext()) {
Object[] row = (Object[]) reader2.readNextRow();
// Default sort column is applied for dimensions. So, need to validate accordingly
- Assert.assertEquals(name[i], row[0]);
- Assert.assertEquals(age[i], row[1]);
+ assert(Arrays.asList(name).contains(row[0]));
+ assert(Arrays.asList(age).contains(row[1]));
i++;
}
- Assert.assertEquals(i, 100);
+ Assert.assertEquals(i, 200);
reader2.close();
FileUtils.deleteDirectory(new File(path));
@@ -114,7 +114,7 @@ public class CarbonReaderTest extends TestCase {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
+ TestUtil.writeFilesAndVerify(100, new Schema(fields), path, true);
CarbonReader reader = CarbonReader
.builder(path, "_temp")
@@ -156,7 +156,7 @@ public class CarbonReaderTest extends TestCase {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
+ TestUtil.writeFilesAndVerify(100, new Schema(fields), path, true);
CarbonReader reader = CarbonReader
.builder(path, "_temp")
@@ -193,7 +193,7 @@ public class CarbonReaderTest extends TestCase {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
+ TestUtil.writeFilesAndVerify(100, new Schema(fields), path, true);
CarbonReader reader = CarbonReader.builder(path, "_temp").isTransactionalTable(true)
.projection(new String[]{"name", "age"}).build();
@@ -233,7 +233,7 @@ public class CarbonReaderTest extends TestCase {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
+ TestUtil.writeFilesAndVerify(100, new Schema(fields), path, true);
CarbonReader reader = CarbonReader
.builder(path)
@@ -309,7 +309,7 @@ public class CarbonReaderTest extends TestCase {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
+ TestUtil.writeFilesAndVerify(100, new Schema(fields), path, true);
File[] dataFiles = new File(path + "/Fact/Part0/Segment_null/").listFiles(new FilenameFilter() {
@Override public boolean accept(File dir, String name) {
@@ -337,7 +337,7 @@ public class CarbonReaderTest extends TestCase {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
+ TestUtil.writeFilesAndVerify(100, new Schema(fields), path, true);
File[] dataFiles = new File(path + "/Metadata").listFiles(new FilenameFilter() {
@Override public boolean accept(File dir, String name) {
@@ -887,7 +887,7 @@ public class CarbonReaderTest extends TestCase {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
+ TestUtil.writeFilesAndVerify(100, new Schema(fields), path, true);
CarbonReader reader = CarbonReader
.builder(path, "_temp")
@@ -926,7 +926,7 @@ public class CarbonReaderTest extends TestCase {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
+ TestUtil.writeFilesAndVerify(100, new Schema(fields), path, true);
CarbonReader reader = CarbonReader
.builder(path, "_temp")
@@ -948,6 +948,7 @@ public class CarbonReaderTest extends TestCase {
Assert.assertEquals(age[i], row[1]);
i++;
}
+ reader.close();
Assert.assertEquals(i, 100);
}
@@ -960,7 +961,7 @@ public class CarbonReaderTest extends TestCase {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
+ TestUtil.writeFilesAndVerify(100, new Schema(fields), path, true);
try {
CarbonReader reader = CarbonReader
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92d9b925/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java
index eb406e2..0f00d61 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java
@@ -39,8 +39,8 @@ public class TestUtil {
writeFilesAndVerify(100, schema, path, sortColumns, false, -1, -1, true);
}
- public static void writeFilesAndVerify(Schema schema, String path, boolean persistSchema) {
- writeFilesAndVerify(100, schema, path, null, persistSchema, -1, -1, true);
+ public static void writeFilesAndVerify(int rows, Schema schema, String path, boolean persistSchema) {
+ writeFilesAndVerify(rows, schema, path, null, persistSchema, -1, -1, true);
}
public static void writeFilesAndVerify(Schema schema, String path, boolean persistSchema,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92d9b925/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java b/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java
index 51d0b27..c885a26 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java
@@ -56,7 +56,7 @@ public class LocalCarbonStoreTest {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
+ TestUtil.writeFilesAndVerify(100, new Schema(fields), path, true);
CarbonStore store = new LocalCarbonStore();
Iterator<CarbonRow> rows = store.scan(path, new String[]{"name, age"}, null);
[39/50] [abbrv] carbondata git commit: [CARBONDATA-2611] Added Test
Cases for Local Dictionary Support for Create Table comand
Posted by ja...@apache.org.
[CARBONDATA-2611] Added Test Cases for Local Dictionary Support for Create Table comand
What changes were proposed in this pull request?
In this PR, UTs and SDV test cases are added for local dictionary support for create table command and describe formatted command.
changed the error message for validation of local dictionary table properties and
fixed column name display without extra space in describe formatted command
This closes#2377
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c5a4ec07
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c5a4ec07
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c5a4ec07
Branch: refs/heads/carbonstore
Commit: c5a4ec07a8c03600741d6a2ae324a4fe4ab61659
Parents: be20fef
Author: praveenmeenakshi56 <pr...@gmail.com>
Authored: Thu Jun 14 23:36:59 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Wed Jun 20 12:13:09 2018 +0530
----------------------------------------------------------------------
...CreateTableWithLocalDictionaryTestCase.scala | 2102 ++++++++++++++++++
.../cluster/sdv/suite/SDVSuites.scala | 6 +-
.../LocalDictionarySupportCreateTableTest.scala | 2102 ++++++++++++++++++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 31 +-
.../table/CarbonDescribeFormattedCommand.scala | 2 +-
5 files changed, 4227 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c5a4ec07/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
new file mode 100644
index 0000000..48a31b7
--- /dev/null
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
@@ -0,0 +1,2102 @@
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+
+class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override protected def beforeAll(): Unit = {
+ sql("DROP TABLE IF EXISTS LOCAL1")
+ }
+
+ test("test local dictionary default configuration") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ val desc_result = sql("describe formatted local1")
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict columns _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='name')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations for local dict columns _002")
+ {
+ sql("drop table if exists local1")
+
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict columns _003") {
+ sql("drop table if exists local1")
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='')
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " +
+ "create table statement"))
+ }
+
+ test("test local dictionary custom configurations for local dict columns _004") {
+ sql("drop table if exists local1")
+ val exception1 = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='abc')
+ """.
+ stripMargin)
+ }
+ assert(exception1.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "create table " +
+ "statement"))
+ }
+
+ test("test local dictionary custom configurations for local dict columns _005") {
+ sql("drop table if exists local1")
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='id')
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_COLUMN should " +
+ "be no dictionary string/complex datatype column"))
+ }
+
+ test("test local dictionary custom configurations for local dict columns _006") {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('dictionary_include'='name','local_dictionary_include'='name')
+ """.
+ stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict threshold _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='10000')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict threshold _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='-100')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict threshold _003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='21474874811')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict threshold _004")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations for local dict threshold _005")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='hello')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_001")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='-100','local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='','local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_004")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='vdslv','local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_005")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_006")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'=' ')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_007")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'='hello')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_008")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'='name',
+ | 'dictionary_include'='name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_009")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='','local_dictionary_include'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_010")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='-100','local_dictionary_include'='Hello')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations with both columns and threshold configured " +
+ "_011")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_threshold'='23213497321591234324',
+ | 'local_dictionary_include'='name','dictionary_include'='name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary default configuration when enabled") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true')
+ """.stripMargin)
+
+ val desc_result = sql("describe formatted local1")
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict columns _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='name','local_dictionary_enable'='true')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict columns _002")
+ {
+ sql("drop table if exists local1")
+
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict columns _003") {
+ sql("drop table if exists local1")
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='')
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " +
+ "create table statement"))
+
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict columns _004") {
+ sql("drop table if exists local1")
+ val exception1 = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='abc')
+ """.
+ stripMargin)
+ }
+ assert(exception1.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "create table " +
+ "statement"))
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict columns _005") {
+ sql("drop table if exists local1")
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='id')
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_COLUMN should " +
+ "be no dictionary string/complex datatype column"))
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict columns _006") {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','dictionary_include'='name',
+ | 'local_dictionary_include'='name')
+ """.
+ stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations when local_dictionary_exclude is configured _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_enable'='true')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test local dictionary custom configurations when local_dictionary_exclude is configured _002")
+ {
+ sql("drop table if exists local1")
+
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary custom configurations when local_dictionary_exclude is configured _003") {
+ sql("drop table if exists local1")
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='')
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " +
+ "create table statement"))
+
+ }
+
+ test("test local dictionary custom configurations when local_dictionary_exclude is configured _004") {
+ sql("drop table if exists local1")
+ val exception1 = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='abc')
+ """.
+ stripMargin)
+ }
+ assert(exception1.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "create table " +
+ "statement"))
+ }
+
+ test("test local dictionary custom configurations when local_dictionary_exclude is configured _005") {
+ sql("drop table if exists local1")
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='id')
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_COLUMN should " +
+ "be no dictionary string/complex datatype column"))
+ }
+
+ test("test local dictionary custom configurations when local_dictionary_exclude is configured _006") {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','dictionary_include'='name',
+ | 'local_dictionary_exclude'='name')
+ """.
+ stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _001")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city',
+ | 'local_dictionary_enable'='true')
+ """.
+ stripMargin)
+
+ val descFormatted1 = sql("describe formatted local1").collect
+
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _002") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int,add string)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city','sort_columns'='add',
+ | 'local_dictionary_enable'='true')
+ """.
+ stripMargin)
+
+ val descFormatted1 = sql("describe formatted local1").collect
+
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city',
+ | 'local_dictionary_enable'='false')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+
+ checkExistence(sql("describe formatted local1"), false, "Local Dictionary Include")
+ checkExistence(sql("describe formatted local1"), false, "Local Dictionary Exclude")
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _004")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city',
+ | 'local_dictionary_enable'='true','dictionary_include'='name,city')
+ """.
+ stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _005")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name,city',
+ | 'local_dictionary_exclude'='name')
+ """.
+ stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _006")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int,st struct<s_id:int,
+ | s_name:string,s_city:array<string>>)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city,st',
+ | 'local_dictionary_enable'='true')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city,st"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _007")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int,st array<struct<s_id:int,
+ | s_name:string>>)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city,st',
+ | 'local_dictionary_enable'='true')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city,st"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict threshold _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict threshold _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict threshold _003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='21474874811')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict threshold _004")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test local dictionary custom configurations when enabled for local dict threshold _005")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='hello')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _001")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _004")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='vdslv',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ sql("desc formatted local1").show(truncate = false)
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _005")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _006")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'=' ')
+ """.stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _007")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='hello')
+ """.stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _008")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='name','dictionary_include'='name')
+ """.stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _009")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='',
+ | 'local_dictionary_include'='name,name')
+ """.stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _010")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100',
+ | 'local_dictionary_include'='Hello')
+ """.stripMargin)
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _011")
+ {
+ sql("drop table if exists local1")
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true',
+ | 'local_dictionary_threshold'='23213497321591234324','local_dictionary_include'='name',
+ | 'dictionary_include'='name')
+ """.stripMargin)
+ }
+ }
+
+ test("test local dictionary default configuration when disabled") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+
+ val desc_result = sql("describe formatted local1")
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict columns _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='name','local_dictionary_enable'='false')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict columns _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_include'='name,name')
+ """.stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict columns _003") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_include'='')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict columns _004") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_include'='abc')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict columns _005") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_include'='id')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict columns _006") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','dictionary_include'='name',
+ | 'local_dictionary_include'='name')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict threshold _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict threshold _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='-100')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict threshold _003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='21474874811')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict threshold _004")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configurations when disabled for local dict threshold _005")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='hello')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _001")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _002")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='-100',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _003")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _004")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='vdslv',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _005")
+ {
+ sql("drop table if exists local1")
+
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='name,name')
+ """.stripMargin)
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _006")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'=' ')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _007")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='hello')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _008")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000',
+ | 'local_dictionary_include'='name','dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _009")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='',
+ | 'local_dictionary_include'='name,name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _010")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='-100',
+ | 'local_dictionary_include'='Hello')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test(
+ "test local dictionary custom configurations when disabled with both columns and threshold " +
+ "configured _011")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='false',
+ | 'local_dictionary_threshold'='23213497321591234324','local_dictionary_include'='name',
+ | 'dictionary_include'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test local dictionary custom configuration with other table properties _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('dictionary_include'='city','sort_scope'='global_sort',
+ | 'sort_columns'='city,name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
+ case Some(row) => assert(row.get(1).toString.contains("global_sort"))
+ }
+ }
+
+ test("test local dictionary custom configuration with other table properties _002") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('dictionary_include'='city','sort_scope'='batch_sort',
+ | 'sort_columns'='city,name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
+ case Some(row) => assert(row.get(1).toString.contains("batch_sort"))
+ }
+ }
+ test("test local dictionary custom configuration with other table properties _003") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('dictionary_include'='city','sort_scope'='no_sort',
+ | 'sort_columns'='city,name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
+ case Some(row) => assert(row.get(1).toString.contains("no_sort"))
+ }
+ }
+ test("test local dictionary custom configuration with other table properties _004") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('dictionary_include'='city','sort_scope'='local_sort',
+ | 'sort_columns'='city,name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
+ case Some(row) => assert(row.get(1).toString.contains("local_sort"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary default configuration when enabled") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | create table local1 stored by 'carbondata' tblproperties('local_dictionary_enable'='true') as
+ | select * from local
+ """.stripMargin)
+
+ val desc_result = sql("describe formatted local1")
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _001") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_include'='name','local_dictionary_enable'='true')
+ | as select * from local
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _002")
+ {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name,name')
+ | as select * from local
+ """.stripMargin)
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _003") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='')
+ | as select * from local
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " +
+ "create table statement"))
+
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _004") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ val exception1 = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='abc')
+ | as select * from local
+ """.
+ stripMargin)
+ }
+ assert(exception1.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "create table " +
+ "statement"))
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _005") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='id')
+ | as select * from local
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_COLUMN should " +
+ "be no dictionary string/complex datatype column"))
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _006") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','dictionary_include'='name',
+ | 'local_dictionary_include'='name') as select * from local
+ """.
+ stripMargin)
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _001") {
+ sql("drop table if exists local")
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_enable'='true')
+ | as select * from local
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _002")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='name,name')
+ | as select * from local
+ """.stripMargin)
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _003") {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='')
+ | as select * from local
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " +
+ "create table statement"))
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _004") {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ val exception1 = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='abc')
+ | as select * from local
+ """.
+ stripMargin)
+ }
+ assert(exception1.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "create table " +
+ "statement"))
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _005") {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ val exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='id')
+ | as select * from local
+ """.
+ stripMargin)
+ }
+ assert(exception.getMessage
+ .contains(
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_COLUMN should " +
+ "be no dictionary string/complex datatype column"))
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _006") {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','dictionary_include'='name',
+ | 'local_dictionary_exclude'='name') as select * from local
+ """.
+ stripMargin)
+ }
+ }
+
+ test(
+ "test CTAS statements for local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _001")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city',
+ | 'local_dictionary_enable'='true') as select * from local
+ """.
+ stripMargin)
+
+ val descFormatted1 = sql("describe formatted local1").collect
+
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test(
+ "test CTAS statements for local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _002")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city',
+ | 'local_dictionary_enable'='false') as select * from local
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+
+ checkExistence(sql("describe formatted local1"), false, "Local Dictionary Include")
+ checkExistence(sql("describe formatted local1"), false, "Local Dictionary Exclude")
+ }
+
+ test(
+ "test CTAS statements for local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _003")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city',
+ | 'local_dictionary_enable'='true','dictionary_include'='name,city') as select * from local
+ """.
+ stripMargin)
+ }
+ }
+
+ test(
+ "test CTAS statements for local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _004")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name,city',
+ | 'local_dictionary_exclude'='name') as select * from local
+ """.
+ stripMargin)
+ }
+ }
+
+ test(
+ "test CTAS statements for local dictionary custom configurations when local_dictionary_include and local_dictionary_exclude " +
+ "is configured _005")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int,st struct<s_id:int,
+ | s_name:string,s_city:array<string>>)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city,st',
+ | 'local_dictionary_enable'='false') as select * from local
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict threshold _001") {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000')
+ | as select * from local
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict threshold _002")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100')
+ | as select * from local
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict threshold _003")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='23589714365172595')
+ | as select * from local
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ }
+
+ test("test CTAS statements for local dictionary custom configurations when first table is hive table")
+ {
+ sql("drop table if exists local1")
+ sql("drop table if exists local")
+ sql(
+ """
+ | CREATE TABLE local(id int, name string, city string, age int)
+ | tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000','local_dictionary_include'='city')
+ | as select * from local
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test("test no inverted index for local dictionary custom configurations when first table is hive table")
+ {
+ sql("drop table if exists local1")
+
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true',
+ | 'local_dictionary_threshold'='10000','local_dictionary_include'='city','no_inverted_index'='name')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("1000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ override protected def afterAll(): Unit = {
+ sql("DROP TABLE IF EXISTS LOCAL1")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c5a4ec07/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
index c5aceaa..6756468 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
@@ -62,7 +62,8 @@ class SDVSuites extends Suites with BeforeAndAfterAll {
new PreAggregateTestCase ::
new TimeSeriesPreAggregateTestCase ::
new TestPartitionWithGlobalSort ::
- new PartitionWithPreAggregateTestCase :: Nil
+ new PartitionWithPreAggregateTestCase ::
+ new CreateTableWithLocalDictionaryTestCase :: Nil
override val nestedSuites = suites.toIndexedSeq
@@ -152,7 +153,8 @@ class SDVSuites3 extends Suites with BeforeAndAfterAll {
new TestPartitionWithGlobalSort ::
new SDKwriterTestCase ::
new SetParameterTestCase ::
- new PartitionWithPreAggregateTestCase :: Nil
+ new PartitionWithPreAggregateTestCase ::
+ new CreateTableWithLocalDictionaryTestCase :: Nil
override val nestedSuites = suites.toIndexedSeq
[41/50] [abbrv] carbondata git commit: [CARBONDATA-2420][32K] Support
string longer than 32000 characters
Posted by ja...@apache.org.
[CARBONDATA-2420][32K] Support string longer than 32000 characters
Add a property in creating table 'long_string_columns' to support string columns that will contains more than 32000 characters.
Inside carbondata, it use an integer instead of short to store the length of bytes content.
Internally in Carbondata,
add a new datatype called varchar to represent the long string column
add a new encoding called DIRECT_COMPRESS_VARCHAR to the varcher column page meta
use an integer (previously short) to store the length of bytes content.
add 2GB constraint for one column page
This closes #2379
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dc53dee2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dc53dee2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dc53dee2
Branch: refs/heads/carbonstore
Commit: dc53dee2448f366319764021d77c4be75d43b9e3
Parents: c5a4ec0
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Sat Jun 2 15:17:04 2018 +0800
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Wed Jun 20 15:24:22 2018 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 3 +
.../impl/FixedLengthDimensionColumnPage.java | 2 +-
.../impl/VariableLengthDimensionColumnPage.java | 11 +-
...mpressedDimensionChunkFileBasedReaderV1.java | 3 +-
...mpressedDimensionChunkFileBasedReaderV2.java | 3 +-
...mpressedDimensionChunkFileBasedReaderV3.java | 7 +-
.../chunk/store/DimensionChunkStoreFactory.java | 22 +-
...ariableIntLengthDimensionDataChunkStore.java | 43 +++
...feVariableLengthDimensionDataChunkStore.java | 45 +--
...iableShortLengthDimensionDataChunkStore.java | 41 +++
...ariableIntLengthDimensionDataChunkStore.java | 44 +++
...feVariableLengthDimensionDataChunkStore.java | 54 ++--
...iableShortLengthDimensionDataChunkStore.java | 44 +++
.../core/datastore/page/ColumnPage.java | 16 +-
.../datastore/page/VarLengthColumnPageBase.java | 6 +
.../page/encoding/DefaultEncodingFactory.java | 1 +
.../page/encoding/EncodingFactory.java | 3 +-
.../encoding/compress/DirectCompressCodec.java | 6 +-
.../legacy/HighCardDictDimensionIndexCodec.java | 13 +-
.../statistics/LVLongStringStatsCollector.java | 51 ++++
.../statistics/LVShortStringStatsCollector.java | 50 ++++
.../page/statistics/LVStringStatsCollector.java | 27 +-
.../core/indexstore/UnsafeMemoryDMStore.java | 11 +-
.../blockletindex/BlockletDataMap.java | 8 +-
.../core/indexstore/row/DataMapRow.java | 4 +-
.../core/indexstore/row/UnsafeDataMapRow.java | 60 ++--
.../core/indexstore/schema/CarbonRowSchema.java | 10 +-
.../core/metadata/blocklet/BlockletInfo.java | 2 +-
.../ThriftWrapperSchemaConverterImpl.java | 8 +
.../core/metadata/datatype/DataType.java | 3 +
.../core/metadata/datatype/DataTypes.java | 5 +
.../core/metadata/datatype/VarcharType.java | 34 +++
.../core/metadata/encoder/Encoding.java | 5 +-
.../schema/table/TableSchemaBuilder.java | 1 +
.../util/AbstractDataFileFooterConverter.java | 2 +
.../apache/carbondata/core/util/CarbonUtil.java | 8 +-
.../carbondata/core/util/DataTypeUtil.java | 4 +
.../ThriftWrapperSchemaConverterImplTest.java | 2 +-
format/src/main/thrift/schema.thrift | 3 +
.../VarcharDataTypesBasicTestCase.scala | 279 +++++++++++++++++++
.../carbondata/spark/util/CarbonScalaUtil.scala | 1 +
.../spark/util/DataTypeConverterUtil.scala | 1 +
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 36 ++-
.../command/carbonTableSchemaCommon.scala | 43 ++-
.../apache/spark/sql/hive/CarbonRelation.scala | 1 +
.../impl/NonDictionaryFieldConverterImpl.java | 12 +-
.../loading/csvinput/CSVInputFormat.java | 4 +-
.../loading/row/IntermediateSortTempRow.java | 19 +-
.../loading/sort/SortStepRowHandler.java | 26 +-
.../merger/CompactionResultSortProcessor.java | 11 +-
.../sort/sortdata/SortParameters.java | 21 +-
.../sort/sortdata/TableFieldStat.java | 37 ++-
.../carbondata/processing/store/TablePage.java | 37 ++-
.../util/CarbonDataProcessorUtil.java | 20 ++
54 files changed, 1049 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 5f06d08..118ff28 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -910,6 +910,7 @@ public final class CarbonCommonConstants {
public static final String COLUMN_GROUPS = "column_groups";
public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
public static final String DICTIONARY_INCLUDE = "dictionary_include";
+ public static final String LONG_STRING_COLUMNS = "long_string_columns";
/**
* Table property to enable or disable local dictionary generation
@@ -1632,6 +1633,8 @@ public final class CarbonCommonConstants {
// As Short data type is used for storing the length of a column during data processing hence
// the maximum characters that can be supported should be less than Short max value
public static final int MAX_CHARS_PER_COLUMN_DEFAULT = 32000;
+ // todo: use infinity first, will switch later
+ public static final int MAX_CHARS_PER_COLUMN_INFINITY = -1;
/**
* Enabling page level reader for compaction reduces the memory usage while compacting more
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
index 76bcf30..570404a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
@@ -47,7 +47,7 @@ public class FixedLengthDimensionColumnPage extends AbstractDimensionColumnPage
dataChunk.length;
dataChunkStore = DimensionChunkStoreFactory.INSTANCE
.getDimensionChunkStore(columnValueSize, isExplicitSorted, numberOfRows, totalSize,
- DimensionStoreType.FIXEDLENGTH);
+ DimensionStoreType.FIXED_LENGTH);
dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunk);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
index 1c6b7f4..7394217 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
@@ -30,21 +30,16 @@ public class VariableLengthDimensionColumnPage extends AbstractDimensionColumnPa
/**
* Constructor for this class
- * @param dataChunks
- * @param invertedIndex
- * @param invertedIndexReverse
- * @param numberOfRows
*/
public VariableLengthDimensionColumnPage(byte[] dataChunks, int[] invertedIndex,
- int[] invertedIndexReverse, int numberOfRows) {
+ int[] invertedIndexReverse, int numberOfRows, DimensionStoreType dimStoreType) {
boolean isExplicitSorted = isExplicitSorted(invertedIndex);
- long totalSize = isExplicitSorted ?
+ long totalSize = null != invertedIndex ?
(dataChunks.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) + (
numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE)) :
(dataChunks.length + (numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE));
dataChunkStore = DimensionChunkStoreFactory.INSTANCE
- .getDimensionChunkStore(0, isExplicitSorted, numberOfRows, totalSize,
- DimensionStoreType.VARIABLELENGTH);
+ .getDimensionChunkStore(0, isExplicitSorted, numberOfRows, totalSize, dimStoreType);
dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunks);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
index 6679402..92a9684 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
@@ -151,7 +152,7 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead
.hasEncoding(dataChunk.getEncodingList(), Encoding.DICTIONARY)) {
columnDataChunk =
new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
- numberOfRows);
+ numberOfRows, DimensionChunkStoreFactory.DimensionStoreType.VARIABLE_SHORT_LENGTH);
} else {
// to store fixed length column chunk values
columnDataChunk =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
index 8938260..3cdbe1d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
@@ -25,6 +25,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReaderV2V3Format;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -175,7 +176,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
if (!hasEncoding(dimensionColumnChunk.encoders, Encoding.DICTIONARY)) {
columnDataChunk =
new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
- numberOfRows);
+ numberOfRows, DimensionChunkStoreFactory.DimensionStoreType.VARIABLE_SHORT_LENGTH);
} else {
// to store fixed length column chunk values
columnDataChunk =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
index 58a9b18..782a8df 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
@@ -27,6 +27,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColum
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReaderV2V3Format;
import org.apache.carbondata.core.datastore.chunk.store.ColumnPageWrapper;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder;
@@ -271,9 +272,13 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
// if no dictionary column then first create a no dictionary column chunk
// and set to data chunk instance
if (!hasEncoding(pageMetadata.encoders, Encoding.DICTIONARY)) {
+ DimensionChunkStoreFactory.DimensionStoreType dimStoreType =
+ hasEncoding(pageMetadata.encoders, Encoding.DIRECT_COMPRESS_VARCHAR) ?
+ DimensionChunkStoreFactory.DimensionStoreType.VARIABLE_INT_LENGTH :
+ DimensionChunkStoreFactory.DimensionStoreType.VARIABLE_SHORT_LENGTH;
columnDataChunk =
new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
- pageMetadata.getNumberOfRowsInpage());
+ pageMetadata.getNumberOfRowsInpage(), dimStoreType);
} else {
// to store fixed length column chunk values
columnDataChunk =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
index f210641..eccfd9c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
@@ -19,9 +19,11 @@ package org.apache.carbondata.core.datastore.chunk.store;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeFixedLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeVariableLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeVariableIntLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeVariableShortLengthDimensionDataChunkStore;
import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeFixedLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeVariableLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeVariableIntLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeVariableShortLengthDimensionDataChunkStore;
import org.apache.carbondata.core.util.CarbonProperties;
/**
@@ -63,19 +65,23 @@ public class DimensionChunkStoreFactory {
boolean isInvertedIndex, int numberOfRows, long totalSize, DimensionStoreType storeType) {
if (isUnsafe) {
- if (storeType == DimensionStoreType.FIXEDLENGTH) {
+ if (storeType == DimensionStoreType.FIXED_LENGTH) {
return new UnsafeFixedLengthDimensionDataChunkStore(totalSize, columnValueSize,
isInvertedIndex, numberOfRows);
+ } else if (storeType == DimensionStoreType.VARIABLE_SHORT_LENGTH) {
+ return new UnsafeVariableShortLengthDimensionDataChunkStore(totalSize, isInvertedIndex,
+ numberOfRows);
} else {
- return new UnsafeVariableLengthDimensionDataChunkStore(totalSize, isInvertedIndex,
+ return new UnsafeVariableIntLengthDimensionDataChunkStore(totalSize, isInvertedIndex,
numberOfRows);
}
-
} else {
- if (storeType == DimensionStoreType.FIXEDLENGTH) {
+ if (storeType == DimensionStoreType.FIXED_LENGTH) {
return new SafeFixedLengthDimensionDataChunkStore(isInvertedIndex, columnValueSize);
+ } else if (storeType == DimensionStoreType.VARIABLE_SHORT_LENGTH) {
+ return new SafeVariableShortLengthDimensionDataChunkStore(isInvertedIndex, numberOfRows);
} else {
- return new SafeVariableLengthDimensionDataChunkStore(isInvertedIndex, numberOfRows);
+ return new SafeVariableIntLengthDimensionDataChunkStore(isInvertedIndex, numberOfRows);
}
}
}
@@ -84,6 +90,6 @@ public class DimensionChunkStoreFactory {
* dimension store type enum
*/
public enum DimensionStoreType {
- FIXEDLENGTH, VARIABLELENGTH;
+ FIXED_LENGTH, VARIABLE_SHORT_LENGTH, VARIABLE_INT_LENGTH;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableIntLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableIntLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableIntLengthDimensionDataChunkStore.java
new file mode 100644
index 0000000..773f078
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableIntLengthDimensionDataChunkStore.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.chunk.store.impl.safe;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+/**
+ * Below class is responsible to store variable long length(>32000) dimension data chunk in
+ * memory. Memory occupied can be on heap or offheap using unsafe interface
+ */
+public class SafeVariableIntLengthDimensionDataChunkStore
+ extends SafeVariableLengthDimensionDataChunkStore {
+ public SafeVariableIntLengthDimensionDataChunkStore(boolean isInvertedIndex, int numberOfRows) {
+ super(isInvertedIndex, numberOfRows);
+ }
+
+ @Override
+ protected int getLengthSize() {
+ return CarbonCommonConstants.INT_SIZE_IN_BYTE;
+ }
+
+ @Override
+ protected int getLengthFromBuffer(ByteBuffer buffer) {
+ return buffer.getInt();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
index bb9c888..52e7317 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
@@ -28,9 +28,10 @@ import org.apache.carbondata.core.util.DataTypeUtil;
/**
* Below class is responsible to store variable length dimension data chunk in
- * memory Memory occupied can be on heap or offheap using unsafe interface
+ * memory. Memory occupied can be on heap or offheap using unsafe interface
*/
-public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimensionDataChunkStore {
+public abstract class SafeVariableLengthDimensionDataChunkStore
+ extends SafeAbsractDimensionDataChunkStore {
/**
* total number of rows
@@ -56,7 +57,8 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
* @param invertedIndexReverse inverted index reverse to be stored
* @param data data to be stored
*/
- @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse,
+ @Override
+ public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse,
byte[] data) {
// first put the data, inverted index and reverse inverted index to memory
super.putArray(invertedIndex, invertedIndexReverse, data);
@@ -75,21 +77,25 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
// as first position will be start from 2 byte as data is stored first in the memory block
// we need to skip first two bytes this is because first two bytes will be length of the data
// which we have to skip
- dataOffsets[0] = CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+ dataOffsets[0] = getLengthSize();
// creating a byte buffer which will wrap the length of the row
ByteBuffer buffer = ByteBuffer.wrap(data);
for (int i = 1; i < numberOfRows; i++) {
buffer.position(startOffset);
// so current row position will be
// previous row length + 2 bytes used for storing previous row data
- startOffset += buffer.getShort() + CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+ startOffset += getLengthFromBuffer(buffer) + getLengthSize();
// as same byte buffer is used to avoid creating many byte buffer for each row
// we need to clear the byte buffer
- dataOffsets[i] = startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+ dataOffsets[i] = startOffset + getLengthSize();
}
}
- @Override public byte[] getRow(int rowId) {
+ protected abstract int getLengthSize();
+ protected abstract int getLengthFromBuffer(ByteBuffer buffer);
+
+ @Override
+ public byte[] getRow(int rowId) {
// if column was explicitly sorted we need to get the rowid based inverted index reverse
if (isExplictSorted) {
rowId = invertedIndexReverse[rowId];
@@ -101,21 +107,21 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
// else subtract the current row offset with complete data
// length get the offset of set of data
int currentDataOffset = dataOffsets[rowId];
- short length = 0;
+ int length = 0;
// calculating the length of data
if (rowId < numberOfRows - 1) {
- length = (short) (dataOffsets[rowId + 1] - (currentDataOffset
- + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+ length = dataOffsets[rowId + 1] - (currentDataOffset + getLengthSize());
} else {
// for last record
- length = (short) (this.data.length - currentDataOffset);
+ length = this.data.length - currentDataOffset;
}
byte[] currentRowData = new byte[length];
System.arraycopy(data, currentDataOffset, currentRowData, 0, length);
return currentRowData;
}
- @Override public void fillRow(int rowId, CarbonColumnVector vector, int vectorRow) {
+ @Override
+ public void fillRow(int rowId, CarbonColumnVector vector, int vectorRow) {
// if column was explicitly sorted we need to get the rowid based inverted index reverse
if (isExplictSorted) {
rowId = invertedIndexReverse[rowId];
@@ -127,11 +133,10 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
// else subtract the current row offset with complete data
// length get the offset of set of data
int currentDataOffset = dataOffsets[rowId];
- short length = 0;
+ int length = 0;
// calculating the length of data
if (rowId < numberOfRows - 1) {
- length = (short) (dataOffsets[rowId + 1] - (currentDataOffset
- + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+ length = dataOffsets[rowId + 1] - (currentDataOffset + getLengthSize());
} else {
// for last record
length = (short) (this.data.length - currentDataOffset);
@@ -162,7 +167,8 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
}
}
- @Override public int compareTo(int rowId, byte[] compareValue) {
+ @Override
+ public int compareTo(int rowId, byte[] compareValue) {
// now to get the row from memory block we need to do following thing
// 1. first get the current offset
// 2. if it's not a last row- get the next row offset
@@ -172,14 +178,13 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
// get the offset of set of data
int currentDataOffset = dataOffsets[rowId];
- short length = 0;
+ int length = 0;
// calculating the length of data
if (rowId < numberOfRows - 1) {
- length = (short) (dataOffsets[rowId + 1] - (currentDataOffset
- + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+ length = dataOffsets[rowId + 1] - (currentDataOffset + getLengthSize());
} else {
// for last record
- length = (short) (this.data.length - currentDataOffset);
+ length = this.data.length - currentDataOffset;
}
return ByteUtil.UnsafeComparer.INSTANCE
.compareTo(data, currentDataOffset, length, compareValue, 0, compareValue.length);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableShortLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableShortLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableShortLengthDimensionDataChunkStore.java
new file mode 100644
index 0000000..beccf86
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableShortLengthDimensionDataChunkStore.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.chunk.store.impl.safe;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+/**
+ * Below class is responsible to store variable long length(>32000) dimension data chunk in
+ * memory. Memory occupied can be on heap or offheap using unsafe interface
+ */
+public class SafeVariableShortLengthDimensionDataChunkStore
+ extends SafeVariableLengthDimensionDataChunkStore {
+ public SafeVariableShortLengthDimensionDataChunkStore(boolean isInvertedIndex, int numberOfRows) {
+ super(isInvertedIndex, numberOfRows);
+ }
+
+ @Override protected int getLengthSize() {
+ return CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+ }
+
+ @Override protected int getLengthFromBuffer(ByteBuffer buffer) {
+ return buffer.getShort();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableIntLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableIntLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableIntLengthDimensionDataChunkStore.java
new file mode 100644
index 0000000..851fff6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableIntLengthDimensionDataChunkStore.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+/**
+ * Below class is responsible to store variable length dimension data chunk in
+ * memory Memory occupied can be on heap or offheap using unsafe interface
+ */
+public class UnsafeVariableIntLengthDimensionDataChunkStore
+ extends UnsafeVariableLengthDimensionDataChunkStore {
+ public UnsafeVariableIntLengthDimensionDataChunkStore(long totalSize, boolean isInvertedIdex,
+ int numberOfRows) {
+ super(totalSize, isInvertedIdex, numberOfRows);
+ }
+
+ @Override
+ protected int getLengthSize() {
+ return CarbonCommonConstants.INT_SIZE_IN_BYTE;
+ }
+
+ @Override
+ protected int getLengthFromBuffer(ByteBuffer byteBuffer) {
+ return byteBuffer.getInt();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimensionDataChunkStore.java
index 07dc806..801a282 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimensionDataChunkStore.java
@@ -31,7 +31,7 @@ import org.apache.carbondata.core.util.DataTypeUtil;
* Below class is responsible to store variable length dimension data chunk in
* memory Memory occupied can be on heap or offheap using unsafe interface
*/
-public class UnsafeVariableLengthDimensionDataChunkStore
+public abstract class UnsafeVariableLengthDimensionDataChunkStore
extends UnsafeAbstractDimensionDataChunkStore {
/**
@@ -67,42 +67,43 @@ public class UnsafeVariableLengthDimensionDataChunkStore
* @param invertedIndexReverse inverted index reverse to be stored
* @param data data to be stored
*/
- @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse,
+ @Override
+ public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse,
byte[] data) {
// first put the data, inverted index and reverse inverted index to memory
super.putArray(invertedIndex, invertedIndexReverse, data);
// position from where offsets will start
this.dataPointersOffsets = this.invertedIndexReverseOffset;
if (isExplicitSorted) {
- this.dataPointersOffsets += (long)numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE;
+ this.dataPointersOffsets += (long) numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE;
}
// As data is of variable length and data format is
- // <length in short><data><length in short><data>
+ // <length in short><data><length in short/int><data>
// we need to store offset of each data so data can be accessed directly
// for example:
//data = {0,5,1,2,3,4,5,0,6,0,1,2,3,4,5,0,2,8,9}
//so value stored in offset will be position of actual data
// [2,9,17]
- // to store this value we need to get the actual data length + 2 bytes used for storing the
+ // to store this value we need to get the actual data length + 2/4 bytes used for storing the
// length
// start position will be used to store the current data position
int startOffset = 0;
- // as first position will be start from 2 byte as data is stored first in the memory block
+ // as first position will be start from 2/4 byte as data is stored first in the memory block
// we need to skip first two bytes this is because first two bytes will be length of the data
// which we have to skip
int [] dataOffsets = new int[numberOfRows];
- dataOffsets[0] = CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+ dataOffsets[0] = getLengthSize();
// creating a byte buffer which will wrap the length of the row
ByteBuffer buffer = ByteBuffer.wrap(data);
for (int i = 1; i < numberOfRows; i++) {
buffer.position(startOffset);
// so current row position will be
- // previous row length + 2 bytes used for storing previous row data
- startOffset += buffer.getShort() + CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+ // previous row length + 2/4 bytes used for storing previous row data
+ startOffset += getLengthFromBuffer(buffer) + getLengthSize();
// as same byte buffer is used to avoid creating many byte buffer for each row
// we need to clear the byte buffer
- dataOffsets[i] = startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+ dataOffsets[i] = startOffset + getLengthSize();
}
CarbonUnsafe.getUnsafe().copyMemory(dataOffsets, CarbonUnsafe.INT_ARRAY_OFFSET,
dataPageMemoryBlock.getBaseObject(),
@@ -110,6 +111,9 @@ public class UnsafeVariableLengthDimensionDataChunkStore
dataOffsets.length * CarbonCommonConstants.INT_SIZE_IN_BYTE);
}
+ protected abstract int getLengthSize();
+ protected abstract int getLengthFromBuffer(ByteBuffer byteBuffer);
+
/**
* Below method will be used to get the row based on row id passed
* Getting the row from unsafe works in below logic
@@ -122,13 +126,14 @@ public class UnsafeVariableLengthDimensionDataChunkStore
* @param rowId
* @return row
*/
- @Override public byte[] getRow(int rowId) {
+ @Override
+ public byte[] getRow(int rowId) {
// get the actual row id
rowId = getRowId(rowId);
// get offset of data in unsafe
int currentDataOffset = getOffSet(rowId);
// get the data length
- short length = getLength(rowId, currentDataOffset);
+ int length = getLength(rowId, currentDataOffset);
// create data array
byte[] data = new byte[length];
// fill the row data
@@ -167,25 +172,24 @@ public class UnsafeVariableLengthDimensionDataChunkStore
/**
* To get the length of data for row id
* if it's not a last row- get the next row offset
- * Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
+ * Subtract the current row offset + 2/4 bytes(to skip the data length) with next row offset
* if it's last row
- * subtract the current row offset + 2 bytes(to skip the data length) with complete data length
+ * subtract the current row offset + 2/4 bytes(to skip the data length) with complete data length
* @param rowId rowId
* @param currentDataOffset current data offset
* @return length of row
*/
- private short getLength(int rowId, int currentDataOffset) {
- short length = 0;
+ private int getLength(int rowId, int currentDataOffset) {
+ int length = 0;
// calculating the length of data
if (rowId < numberOfRows - 1) {
int OffsetOfNextdata = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((rowId + 1)
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
- length = (short) (OffsetOfNextdata - (currentDataOffset
- + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+ length = OffsetOfNextdata - (currentDataOffset + getLengthSize());
} else {
// for last record we need to subtract with data length
- length = (short) (this.dataLength - currentDataOffset);
+ length = this.dataLength - currentDataOffset;
}
return length;
}
@@ -196,7 +200,7 @@ public class UnsafeVariableLengthDimensionDataChunkStore
* @param data data array
* @param currentDataOffset current data offset
*/
- private void fillRowInternal(short length, byte[] data, int currentDataOffset) {
+ private void fillRowInternal(int length, byte[] data, int currentDataOffset) {
CarbonUnsafe.getUnsafe().copyMemory(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + currentDataOffset, data,
CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
@@ -217,13 +221,14 @@ public class UnsafeVariableLengthDimensionDataChunkStore
* @param vectorRow vector row id
*
*/
- @Override public void fillRow(int rowId, CarbonColumnVector vector, int vectorRow) {
+ @Override
+ public void fillRow(int rowId, CarbonColumnVector vector, int vectorRow) {
// get the row id from reverse inverted index based on row id
rowId = getRowId(rowId);
// get the current row offset
int currentDataOffset = getOffSet(rowId);
// get the row data length
- short length = getLength(rowId, currentDataOffset);
+ int length = getLength(rowId, currentDataOffset);
// check if value length is less the current data length
// then create a new array else use the same
if (length > value.length) {
@@ -262,9 +267,10 @@ public class UnsafeVariableLengthDimensionDataChunkStore
* @param compareValue value of to be compared
* @return compare result
*/
- @Override public int compareTo(int rowId, byte[] compareValue) {
+ @Override
+ public int compareTo(int rowId, byte[] compareValue) {
int currentDataOffset = getOffSet(rowId);;
- short length = getLength(rowId, currentDataOffset);
+ int length = getLength(rowId, currentDataOffset);
// as this class handles this variable length data, so filter value can be
// smaller or bigger than than actual data, so we need to take the smaller length
int compareResult;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableShortLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableShortLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableShortLengthDimensionDataChunkStore.java
new file mode 100644
index 0000000..995f5ba
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableShortLengthDimensionDataChunkStore.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+/**
+ * Below class is responsible to store variable length dimension data chunk in
+ * memory Memory occupied can be on heap or offheap using unsafe interface
+ */
+public class UnsafeVariableShortLengthDimensionDataChunkStore
+ extends UnsafeVariableLengthDimensionDataChunkStore {
+ public UnsafeVariableShortLengthDimensionDataChunkStore(long totalSize, boolean isInvertedIdex,
+ int numberOfRows) {
+ super(totalSize, isInvertedIdex, numberOfRows);
+ }
+
+ @Override
+ protected int getLengthSize() {
+ return CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+ }
+
+ @Override
+ protected int getLengthFromBuffer(ByteBuffer byteBuffer) {
+ return byteBuffer.getShort();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index 69ed437..4dcf514 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -203,7 +203,9 @@ public abstract class ColumnPage {
instance = new UnsafeFixLengthColumnPage(columnSpec, dataType, pageSize);
} else if (DataTypes.isDecimal(dataType)) {
instance = new UnsafeDecimalColumnPage(columnSpec, dataType, pageSize);
- } else if (dataType == DataTypes.STRING || dataType == DataTypes.BYTE_ARRAY) {
+ } else if (dataType == DataTypes.STRING
+ || dataType == DataTypes.BYTE_ARRAY
+ || dataType == DataTypes.VARCHAR) {
instance = new UnsafeVarLengthColumnPage(columnSpec, dataType, pageSize);
} else {
throw new RuntimeException("Unsupported data dataType: " + dataType);
@@ -225,7 +227,9 @@ public abstract class ColumnPage {
instance = newDoublePage(columnSpec, new double[pageSize]);
} else if (DataTypes.isDecimal(dataType)) {
instance = newDecimalPage(columnSpec, new byte[pageSize][]);
- } else if (dataType == DataTypes.STRING || dataType == DataTypes.BYTE_ARRAY) {
+ } else if (dataType == DataTypes.STRING
+ || dataType == DataTypes.BYTE_ARRAY
+ || dataType == DataTypes.VARCHAR) {
instance = new SafeVarLengthColumnPage(columnSpec, dataType, pageSize);
} else {
throw new RuntimeException("Unsupported data dataType: " + dataType);
@@ -398,7 +402,9 @@ public abstract class ColumnPage {
} else if (DataTypes.isDecimal(dataType)) {
putDecimal(rowId, (BigDecimal) value);
statsCollector.update((BigDecimal) value);
- } else if (dataType == DataTypes.STRING || dataType == DataTypes.BYTE_ARRAY) {
+ } else if (dataType == DataTypes.STRING
+ || dataType == DataTypes.BYTE_ARRAY
+ || dataType == DataTypes.VARCHAR) {
putBytes(rowId, (byte[]) value);
statsCollector.update((byte[]) value);
} else {
@@ -431,7 +437,9 @@ public abstract class ColumnPage {
return getDouble(rowId);
} else if (DataTypes.isDecimal(dataType)) {
return getDecimal(rowId);
- } else if (dataType == DataTypes.STRING || dataType == DataTypes.BYTE_ARRAY) {
+ } else if (dataType == DataTypes.STRING
+ || dataType == DataTypes.BYTE_ARRAY
+ || dataType == DataTypes.VARCHAR) {
return getBytes(rowId);
} else {
throw new RuntimeException("unsupported data type: " + dataType);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
index 901758a..cb907a5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
@@ -289,6 +289,12 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
@Override
public void putBytes(int rowId, byte[] bytes) {
+ // rowId * 4 represents the length of L in LV
+ if (bytes.length > (Integer.MAX_VALUE - totalLength - rowId * 4)) {
+ // since we later store a column page in a byte array, so its maximum size is 2GB
+ throw new RuntimeException("Carbondata only support maximum 2GB size for one column page,"
+ + " exceed this limit at rowId " + rowId);
+ }
if (rowId == 0) {
rowOffset[0] = 0;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
index 00f7a0f..816b01f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
@@ -103,6 +103,7 @@ public class DefaultEncodingFactory extends EncodingFactory {
return new HighCardDictDimensionIndexCodec(
dimensionSpec.isInSortColumns(),
dimensionSpec.isInSortColumns() && dimensionSpec.isDoInvertedIndex(),
+ dimensionSpec.getSchemaDataType() == DataTypes.VARCHAR,
compressor).createEncoder(null);
default:
throw new RuntimeException("unsupported dimension type: " +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
index 318d55d..a661a49 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
@@ -47,6 +47,7 @@ import static org.apache.carbondata.format.Encoding.ADAPTIVE_FLOATING;
import static org.apache.carbondata.format.Encoding.ADAPTIVE_INTEGRAL;
import static org.apache.carbondata.format.Encoding.BOOL_BYTE;
import static org.apache.carbondata.format.Encoding.DIRECT_COMPRESS;
+import static org.apache.carbondata.format.Encoding.DIRECT_COMPRESS_VARCHAR;
import static org.apache.carbondata.format.Encoding.RLE_INTEGRAL;
/**
@@ -71,7 +72,7 @@ public abstract class EncodingFactory {
byte[] encoderMeta = encoderMetas.get(0).array();
ByteArrayInputStream stream = new ByteArrayInputStream(encoderMeta);
DataInputStream in = new DataInputStream(stream);
- if (encoding == DIRECT_COMPRESS) {
+ if (encoding == DIRECT_COMPRESS || encoding == DIRECT_COMPRESS_VARCHAR) {
ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
metadata.readFields(in);
return new DirectCompressCodec(metadata.getStoreDataType()).createDecoder(metadata);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
index cfdf114..4c1bc49 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
@@ -64,7 +64,7 @@ public class DirectCompressCodec implements ColumnPageCodec {
return new DirectDecompressor(meta);
}
- private static class DirectCompressor extends ColumnPageEncoder {
+ private class DirectCompressor extends ColumnPageEncoder {
private Compressor compressor;
@@ -80,7 +80,9 @@ public class DirectCompressCodec implements ColumnPageCodec {
@Override
protected List<Encoding> getEncodingList() {
List<Encoding> encodings = new ArrayList<>();
- encodings.add(Encoding.DIRECT_COMPRESS);
+ encodings.add(dataType == DataTypes.VARCHAR ?
+ Encoding.DIRECT_COMPRESS_VARCHAR :
+ Encoding.DIRECT_COMPRESS);
return encodings;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/HighCardDictDimensionIndexCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/HighCardDictDimensionIndexCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/HighCardDictDimensionIndexCodec.java
index d722c38..741dbfe 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/HighCardDictDimensionIndexCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/HighCardDictDimensionIndexCodec.java
@@ -30,11 +30,16 @@ import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoder;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.format.Encoding;
-public class HighCardDictDimensionIndexCodec extends IndexStorageCodec {
+public class HighCardDictDimensionIndexCodec extends IndexStorageCodec {
+ /**
+ * whether this column is varchar data type(long string)
+ */
+ private boolean isVarcharType;
public HighCardDictDimensionIndexCodec(boolean isSort, boolean isInvertedIndex,
- Compressor compressor) {
+ boolean isVarcharType, Compressor compressor) {
super(isSort, isInvertedIndex, compressor);
+ this.isVarcharType = isVarcharType;
}
@Override
@@ -63,7 +68,9 @@ public class HighCardDictDimensionIndexCodec extends IndexStorageCodec {
@Override
protected List<Encoding> getEncodingList() {
List<Encoding> encodings = new ArrayList<>();
- if (indexStorage.getRowIdPageLengthInBytes() > 0) {
+ if (isVarcharType) {
+ encodings.add(Encoding.DIRECT_COMPRESS_VARCHAR);
+ } else if (indexStorage.getRowIdPageLengthInBytes() > 0) {
encodings.add(Encoding.INVERTED_INDEX);
}
return encodings;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVLongStringStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVLongStringStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVLongStringStatsCollector.java
new file mode 100644
index 0000000..a7bb47e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVLongStringStatsCollector.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.statistics;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * This class is for the columns with varchar data type,
+ * a string type which can hold more than 32000 characters
+ */
+public class LVLongStringStatsCollector extends LVStringStatsCollector {
+
+ public static LVLongStringStatsCollector newInstance() {
+ return new LVLongStringStatsCollector();
+ }
+
+ private LVLongStringStatsCollector() {
+
+ }
+
+ @Override
+ protected byte[] getActualValue(byte[] value) {
+ byte[] actualValue;
+ assert (value.length >= 4);
+ if (value.length == 4) {
+ assert (value[0] == 0 && value[1] == 0);
+ actualValue = new byte[0];
+ } else {
+ int length = ByteUtil.toInt(value, 0);
+ assert (length > 0);
+ actualValue = new byte[value.length - 4];
+ System.arraycopy(value, 4, actualValue, 0, actualValue.length);
+ }
+ return actualValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVShortStringStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVShortStringStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVShortStringStatsCollector.java
new file mode 100644
index 0000000..21b06d5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVShortStringStatsCollector.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.statistics;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * This class is for the columns with string data type which hold less than 32000 characters
+ */
+public class LVShortStringStatsCollector extends LVStringStatsCollector {
+
+ public static LVShortStringStatsCollector newInstance() {
+ return new LVShortStringStatsCollector();
+ }
+
+ private LVShortStringStatsCollector() {
+
+ }
+
+ @Override
+ protected byte[] getActualValue(byte[] value) {
+ byte[] actualValue;
+ assert (value.length >= 2);
+ if (value.length == 2) {
+ assert (value[0] == 0 && value[1] == 0);
+ actualValue = new byte[0];
+ } else {
+ int length = ByteUtil.toShort(value, 0);
+ assert (length > 0);
+ actualValue = new byte[value.length - 2];
+ System.arraycopy(value, 2, actualValue, 0, actualValue.length);
+ }
+ return actualValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
index 7958a8d..e1ac676 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
@@ -23,18 +23,10 @@ import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.ByteUtil;
-public class LVStringStatsCollector implements ColumnPageStatsCollector {
+public abstract class LVStringStatsCollector implements ColumnPageStatsCollector {
private byte[] min, max;
- public static LVStringStatsCollector newInstance() {
- return new LVStringStatsCollector();
- }
-
- private LVStringStatsCollector() {
-
- }
-
@Override
public void updateNull(int rowId) {
@@ -70,22 +62,13 @@ public class LVStringStatsCollector implements ColumnPageStatsCollector {
}
+ protected abstract byte[] getActualValue(byte[] value);
+
@Override
public void update(byte[] value) {
// input value is LV encoded
- byte[] newValue = null;
- assert (value.length >= 2);
- if (value.length == 2) {
- assert (value[0] == 0 && value[1] == 0);
- newValue = new byte[0];
- } else {
- int length = (value[0] << 8) + (value[1] & 0xff);
- assert (length > 0);
- newValue = new byte[value.length - 2];
- System.arraycopy(value, 2, newValue, 0, newValue.length);
- }
-
- if (null == min) {
+ byte[] newValue = getActualValue(value);
+ if (min == null) {
min = newValue;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
index ca5e2dd..599877c 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
@@ -144,7 +144,7 @@ public class UnsafeMemoryDMStore extends AbstractMemoryDMStore {
"unsupported data type for unsafe storage: " + schema.getDataType());
}
break;
- case VARIABLE:
+ case VARIABLE_SHORT:
byte[] data = row.getByteArray(index);
getUnsafe().putShort(memoryBlock.getBaseObject(),
memoryBlock.getBaseOffset() + runningLength, (short) data.length);
@@ -153,6 +153,15 @@ public class UnsafeMemoryDMStore extends AbstractMemoryDMStore {
memoryBlock.getBaseOffset() + runningLength, data.length);
runningLength += data.length;
break;
+ case VARIABLE_INT:
+ byte[] data2 = row.getByteArray(index);
+ getUnsafe().putInt(memoryBlock.getBaseObject(),
+ memoryBlock.getBaseOffset() + runningLength, data2.length);
+ runningLength += 4;
+ getUnsafe().copyMemory(data2, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(),
+ memoryBlock.getBaseOffset() + runningLength, data2.length);
+ runningLength += data2.length;
+ break;
case STRUCT:
CarbonRowSchema[] childSchemas =
((CarbonRowSchema.StructCarbonRowSchema) schema).getChildSchemas();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 6e43fbc..4b5b36b 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -607,7 +607,13 @@ public class BlockletDataMap extends CoarseGrainDataMap implements Serializable
CarbonRowSchema[] mapSchemas = new CarbonRowSchema[minMaxLen.length];
for (int i = 0; i < minMaxLen.length; i++) {
if (minMaxLen[i] <= 0) {
- mapSchemas[i] = new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY);
+ boolean isVarchar = false;
+ if (i < segmentProperties.getDimensions().size()
+ && segmentProperties.getDimensions().get(i).getDataType() == DataTypes.VARCHAR) {
+ isVarchar = true;
+ }
+ mapSchemas[i] = new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY,
+ isVarchar);
} else {
mapSchemas[i] =
new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, minMaxLen[i]);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java
index 496a1d0..b8b46ef 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java
@@ -78,8 +78,10 @@ public abstract class DataMapRow implements Serializable {
switch (schemas[ordinal].getSchemaType()) {
case FIXED:
return schemas[ordinal].getLength();
- case VARIABLE:
+ case VARIABLE_SHORT:
return getLengthInBytes(ordinal) + 2;
+ case VARIABLE_INT:
+ return getLengthInBytes(ordinal) + 4;
case STRUCT:
return getRow(ordinal).getTotalSizeInBytes();
default:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java b/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
index 1c1ecad..127e2a9 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
@@ -49,11 +49,16 @@ public class UnsafeDataMapRow extends DataMapRow {
int length;
int position = getPosition(ordinal);
switch (schemas[ordinal].getSchemaType()) {
- case VARIABLE:
- length =
- getUnsafe().getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
+ case VARIABLE_SHORT:
+ length = getUnsafe().getShort(block.getBaseObject(),
+ block.getBaseOffset() + pointer + position);
position += 2;
break;
+ case VARIABLE_INT:
+ length = getUnsafe().getInt(block.getBaseObject(),
+ block.getBaseOffset() + pointer + position);
+ position += 4;
+ break;
default:
length = schemas[ordinal].getLength();
}
@@ -67,9 +72,13 @@ public class UnsafeDataMapRow extends DataMapRow {
int length;
int position = getPosition(ordinal);
switch (schemas[ordinal].getSchemaType()) {
- case VARIABLE:
- length =
- getUnsafe().getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
+ case VARIABLE_SHORT:
+ length = getUnsafe().getShort(block.getBaseObject(),
+ block.getBaseOffset() + pointer + position);
+ break;
+ case VARIABLE_INT:
+ length = getUnsafe().getInt(block.getBaseObject(),
+ block.getBaseOffset() + pointer + position);
break;
default:
length = schemas[ordinal].getLength();
@@ -80,9 +89,13 @@ public class UnsafeDataMapRow extends DataMapRow {
private int getLengthInBytes(int ordinal, int position) {
int length;
switch (schemas[ordinal].getSchemaType()) {
- case VARIABLE:
- length =
- getUnsafe().getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
+ case VARIABLE_SHORT:
+ length = getUnsafe().getShort(block.getBaseObject(),
+ block.getBaseOffset() + pointer + position);
+ break;
+ case VARIABLE_INT:
+ length = getUnsafe().getInt(block.getBaseObject(),
+ block.getBaseOffset() + pointer + position);
break;
default:
length = schemas[ordinal].getLength();
@@ -226,21 +239,28 @@ public class UnsafeDataMapRow extends DataMapRow {
"unsupported data type for unsafe storage: " + schema.getDataType());
}
break;
- case VARIABLE:
- short length = getUnsafe().getShort(
- block.getBaseObject(),
- block.getBaseOffset() + pointer + runningLength);
+ case VARIABLE_SHORT:
+ int length = getUnsafe()
+ .getShort(block.getBaseObject(), block.getBaseOffset() + pointer + runningLength);
runningLength += 2;
byte[] data = new byte[length];
- getUnsafe().copyMemory(
- block.getBaseObject(),
+ getUnsafe().copyMemory(block.getBaseObject(),
block.getBaseOffset() + pointer + runningLength,
- data,
- BYTE_ARRAY_OFFSET,
- data.length);
+ data, BYTE_ARRAY_OFFSET, data.length);
runningLength += data.length;
row.setByteArray(data, i);
break;
+ case VARIABLE_INT:
+ int length2 = getUnsafe()
+ .getInt(block.getBaseObject(), block.getBaseOffset() + pointer + runningLength);
+ runningLength += 4;
+ byte[] data2 = new byte[length2];
+ getUnsafe().copyMemory(block.getBaseObject(),
+ block.getBaseOffset() + pointer + runningLength,
+ data2, BYTE_ARRAY_OFFSET, data2.length);
+ runningLength += data2.length;
+ row.setByteArray(data2, i);
+ break;
case STRUCT:
DataMapRow structRow = ((UnsafeDataMapRow) getRow(i)).convertToSafeRow();
row.setRow(structRow, i);
@@ -260,8 +280,10 @@ public class UnsafeDataMapRow extends DataMapRow {
switch (schemas[ordinal].getSchemaType()) {
case FIXED:
return schemas[ordinal].getLength();
- case VARIABLE:
+ case VARIABLE_SHORT:
return getLengthInBytes(ordinal, position) + 2;
+ case VARIABLE_INT:
+ return getLengthInBytes(ordinal, position) + 4;
case STRUCT:
return getRow(ordinal).getTotalSizeInBytes();
default:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/indexstore/schema/CarbonRowSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/CarbonRowSchema.java b/core/src/main/java/org/apache/carbondata/core/indexstore/schema/CarbonRowSchema.java
index 1a77467..971f42a 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/CarbonRowSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/schema/CarbonRowSchema.java
@@ -90,17 +90,23 @@ public abstract class CarbonRowSchema implements Serializable {
}
public static class VariableCarbonRowSchema extends CarbonRowSchema {
+ private boolean isVarcharType = false;
public VariableCarbonRowSchema(DataType dataType) {
super(dataType);
}
+ public VariableCarbonRowSchema(DataType dataType, boolean isVarcharType) {
+ super(dataType);
+ this.isVarcharType = isVarcharType;
+ }
+
@Override public int getLength() {
return dataType.getSizeInBytes();
}
@Override public DataMapSchemaType getSchemaType() {
- return DataMapSchemaType.VARIABLE;
+ return isVarcharType ? DataMapSchemaType.VARIABLE_INT : DataMapSchemaType.VARIABLE_SHORT;
}
}
@@ -127,6 +133,6 @@ public abstract class CarbonRowSchema implements Serializable {
}
public enum DataMapSchemaType {
- FIXED, VARIABLE, STRUCT
+ FIXED, VARIABLE_INT, VARIABLE_SHORT, STRUCT
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
index f77358f..420cd4e 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
@@ -268,7 +268,7 @@ public class BlockletInfo implements Serializable, Writable {
@Override public void readFields(DataInput input) throws IOException {
dimensionOffset = input.readLong();
measureOffsets = input.readLong();
- short dimensionChunkOffsetsSize = input.readShort();
+ int dimensionChunkOffsetsSize = input.readShort();
dimensionChunkOffsets = new ArrayList<>(dimensionChunkOffsetsSize);
for (int i = 0; i < dimensionChunkOffsetsSize; i++) {
dimensionChunkOffsets.add(input.readLong());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 12f5fc3..87dda33 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -112,6 +112,8 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return org.apache.carbondata.format.Encoding.RLE;
case INVERTED_INDEX:
return org.apache.carbondata.format.Encoding.INVERTED_INDEX;
+ case DIRECT_COMPRESS_VARCHAR:
+ return org.apache.carbondata.format.Encoding.DIRECT_COMPRESS_VARCHAR;
case BIT_PACKED:
return org.apache.carbondata.format.Encoding.BIT_PACKED;
case DIRECT_DICTIONARY:
@@ -154,6 +156,8 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return org.apache.carbondata.format.DataType.ARRAY;
} else if (DataTypes.isStructType(dataType)) {
return org.apache.carbondata.format.DataType.STRUCT;
+ } else if (dataType.getId() == DataTypes.VARCHAR.getId()) {
+ return org.apache.carbondata.format.DataType.VARCHAR;
} else {
return org.apache.carbondata.format.DataType.STRING;
}
@@ -447,6 +451,8 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return Encoding.RLE;
case INVERTED_INDEX:
return Encoding.INVERTED_INDEX;
+ case DIRECT_COMPRESS_VARCHAR:
+ return Encoding.DIRECT_COMPRESS_VARCHAR;
case BIT_PACKED:
return Encoding.BIT_PACKED;
case DIRECT_DICTIONARY:
@@ -490,6 +496,8 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return DataTypes.createDefaultArrayType();
case STRUCT:
return DataTypes.createDefaultStructType();
+ case VARCHAR:
+ return DataTypes.VARCHAR;
default:
return DataTypes.STRING;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
index d71f984..4dc1fbc 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
@@ -69,6 +69,7 @@ public class DataType implements Serializable {
public static final char DOUBLE_MEASURE_CHAR = 'n';
public static final char STRING_CHAR = 's';
+ public static final char VARCHAR_CHAR = 'v';
public static final char TIMESTAMP_CHAR = 't';
public static final char DATE_CHAR = 'x';
public static final char BYTE_ARRAY_CHAR = 'y';
@@ -89,6 +90,8 @@ public class DataType implements Serializable {
return BIG_DECIMAL_MEASURE_CHAR;
} else if (dataType == DataTypes.STRING) {
return STRING_CHAR;
+ } else if (dataType == DataTypes.VARCHAR) {
+ return VARCHAR_CHAR;
} else if (dataType == DataTypes.TIMESTAMP) {
return TIMESTAMP_CHAR;
} else if (dataType == DataTypes.DATE) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
index dc89a41..d71eea4 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
@@ -47,6 +47,8 @@ public class DataTypes {
// Only for internal use for backward compatability. It is only used for V1 version
public static final DataType LEGACY_LONG = LegacyLongType.LEGACY_LONG;
+ public static final DataType VARCHAR = VarcharType.VARCHAR;
+
// these IDs are used within this package only
static final int STRING_TYPE_ID = 0;
static final int DATE_TYPE_ID = 1;
@@ -66,6 +68,7 @@ public class DataTypes {
public static final int ARRAY_TYPE_ID = 11;
public static final int STRUCT_TYPE_ID = 12;
public static final int MAP_TYPE_ID = 13;
+ public static final int VARCHAR_TYPE_ID = 18;
/**
* create a DataType instance from uniqueId of the DataType
@@ -107,6 +110,8 @@ public class DataTypes {
return createDefaultMapType();
} else if (id == BYTE_ARRAY.getId()) {
return BYTE_ARRAY;
+ } else if (id == VARCHAR.getId()) {
+ return VARCHAR;
} else {
throw new RuntimeException("create DataType with invalid id: " + id);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/metadata/datatype/VarcharType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/VarcharType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/VarcharType.java
new file mode 100644
index 0000000..bfde1a9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/VarcharType.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.datatype;
+
+/**
+ * This class is for internal use. It is used to support string that longer than 32000 characters
+ */
+public class VarcharType extends DataType {
+ static final DataType VARCHAR = new VarcharType(DataTypes.VARCHAR_TYPE_ID, 0, "VARCHAR", -1);
+
+ private VarcharType(int id, int precedenceOrder, String name, int sizeInBytes) {
+ super(id, precedenceOrder, name, sizeInBytes);
+ }
+
+ // this function is needed to ensure singleton pattern while supporting java serialization
+ private Object readResolve() {
+ return DataTypes.VARCHAR;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/metadata/encoder/Encoding.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/encoder/Encoding.java b/core/src/main/java/org/apache/carbondata/core/metadata/encoder/Encoding.java
index 06d09f8..f3c21b1 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/encoder/Encoding.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/encoder/Encoding.java
@@ -31,7 +31,8 @@ public enum Encoding {
DIRECT_COMPRESS,
ADAPTIVE_INTEGRAL,
ADAPTIVE_DELTA_INTEGRAL,
- RLE_INTEGRAL;
+ RLE_INTEGRAL,
+ DIRECT_COMPRESS_VARCHAR;
public static Encoding valueOf(int ordinal) {
if (ordinal == DICTIONARY.ordinal()) {
@@ -56,6 +57,8 @@ public enum Encoding {
return ADAPTIVE_DELTA_INTEGRAL;
} else if (ordinal == RLE_INTEGRAL.ordinal()) {
return RLE_INTEGRAL;
+ } else if (ordinal == DIRECT_COMPRESS_VARCHAR.ordinal()) {
+ return DIRECT_COMPRESS_VARCHAR;
} else {
throw new RuntimeException("create Encoding with invalid ordinal: " + ordinal);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java
index bb7e901..40f8725 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java
@@ -203,6 +203,7 @@ public class TableSchemaBuilder {
}
}
}
+ // todo: need more information such as long_string_columns
return newColumn;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index f005d88..7cd0c18 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -436,6 +436,8 @@ public abstract class AbstractDataFileFooterConverter {
return DataTypes.createDefaultArrayType();
case STRUCT:
return DataTypes.createDefaultStructType();
+ case VARCHAR:
+ return DataTypes.VARCHAR;
default:
return DataTypes.STRING;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 2f34163..1f6c697 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2309,6 +2309,8 @@ public final class CarbonUtil {
return DataTypes.createDefaultArrayType();
case STRUCT:
return DataTypes.createDefaultStructType();
+ case VARCHAR:
+ return DataTypes.VARCHAR;
default:
return DataTypes.STRING;
}
@@ -2499,8 +2501,10 @@ public final class CarbonUtil {
return DataTypeUtil.bigDecimalToByte((BigDecimal) value);
} else if (dataType == DataTypes.BYTE_ARRAY) {
return (byte[]) value;
- } else if (dataType == DataTypes.STRING || dataType == DataTypes.TIMESTAMP ||
- dataType == DataTypes.DATE) {
+ } else if (dataType == DataTypes.STRING
+ || dataType == DataTypes.TIMESTAMP
+ || dataType == DataTypes.DATE
+ || dataType == DataTypes.VARCHAR) {
return (byte[]) value;
} else {
throw new IllegalArgumentException("Invalid data type: " + dataType);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index e06c82e..c84b0da 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -856,6 +856,8 @@ public final class DataTypeUtil {
return DataTypes.FLOAT;
} else if (DataTypes.DOUBLE.getName().equalsIgnoreCase(name)) {
return DataTypes.DOUBLE;
+ } else if (DataTypes.VARCHAR.getName().equalsIgnoreCase(name)) {
+ return DataTypes.VARCHAR;
} else if (DataTypes.NULL.getName().equalsIgnoreCase(name)) {
return DataTypes.NULL;
} else if (DataTypes.BYTE_ARRAY.getName().equalsIgnoreCase(name)) {
@@ -904,6 +906,8 @@ public final class DataTypeUtil {
return DataTypes.FLOAT;
} else if (DataTypes.DOUBLE.getName().equalsIgnoreCase(dataType.getName())) {
return DataTypes.DOUBLE;
+ } else if (DataTypes.VARCHAR.getName().equalsIgnoreCase(dataType.getName())) {
+ return DataTypes.VARCHAR;
} else if (DataTypes.NULL.getName().equalsIgnoreCase(dataType.getName())) {
return DataTypes.NULL;
} else if (DataTypes.BYTE_ARRAY.getName().equalsIgnoreCase(dataType.getName())) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
index 67c7594..522bf41 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
@@ -1562,7 +1562,7 @@ public class ThriftWrapperSchemaConverterImplTest {
}
@Test public void testFromExternalToWrapperSchemaEvolutionEntry() {
-long time =1112745600000L;
+ long time =1112745600000L;
ColumnSchema wrapperColumnSchema = new ColumnSchema();
wrapperColumnSchema.setColumnUniqueId("1");
wrapperColumnSchema.setColumnName("columnName");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/format/src/main/thrift/schema.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index b44fe19..3af2b9a 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -35,6 +35,7 @@ enum DataType {
BOOLEAN = 8,
ARRAY = 20,
STRUCT = 21,
+ VARCHAR = 22,
}
/**
@@ -56,6 +57,7 @@ enum Encoding{
ADAPTIVE_FLOATING = 11; // Identifies that a column is encoded using AdaptiveFloatingCodec
BOOL_BYTE = 12; // Identifies that a column is encoded using BooleanPageCodec
ADAPTIVE_DELTA_FLOATING = 13; // Identifies that a column is encoded using AdaptiveDeltaFloatingCodec
+ DIRECT_COMPRESS_VARCHAR = 14; // Identifies that a columm is encoded using DirectCompressCodec, it is used for long string columns
}
enum PartitionType{
@@ -173,6 +175,7 @@ struct TableSchema{
4: optional map<string,string> tableProperties; // Table properties configured by the user
5: optional BucketingInfo bucketingInfo; // Bucketing information
6: optional PartitionInfo partitionInfo; // Partition information
+ 7: optional list<string> long_string_columns // long string columns in the table
}
struct RelationIdentifier {
[12/50] [abbrv] carbondata git commit: [CARBONDATA-2575] Add document
to explain DataMap Management
Posted by ja...@apache.org.
[CARBONDATA-2575] Add document to explain DataMap Management
Add document to explain DataMap Management
This closes #2360
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d401e060
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d401e060
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d401e060
Branch: refs/heads/carbonstore
Commit: d401e060adcc531d66468dc61f4d468768cfea3f
Parents: 5f68a79
Author: Jacky Li <ja...@qq.com>
Authored: Mon Jun 4 21:18:31 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Tue Jun 5 19:57:06 2018 +0800
----------------------------------------------------------------------
docs/datamap/datamap-management.md | 111 ++++++++++++++++++++++++++++++++
1 file changed, 111 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d401e060/docs/datamap/datamap-management.md
----------------------------------------------------------------------
diff --git a/docs/datamap/datamap-management.md b/docs/datamap/datamap-management.md
new file mode 100644
index 0000000..01bb69f
--- /dev/null
+++ b/docs/datamap/datamap-management.md
@@ -0,0 +1,111 @@
+# CarbonData DataMap Management
+
+## Overview
+
+DataMap can be created using following DDL
+
+```
+ CREATE DATAMAP [IF NOT EXISTS] datamap_name
+ [ON TABLE main_table]
+ USING "datamap_provider"
+ [WITH DEFERRED REBUILD]
+ DMPROPERTIES ('key'='value', ...)
+ AS
+ SELECT statement
+```
+
+Currently, there are 5 DataMap implementation in CarbonData.
+
+| DataMap Provider | Description | DMPROPERTIES | Management |
+| ---------------- | ---------------------------------------- | ---------------------------------------- | ---------------- |
+| preaggregate | single table pre-aggregate table | No DMPROPERTY is required | Automatic |
+| timeseries | time dimension rollup table. | event_time, xx_granularity, please refer to [Timeseries DataMap](https://github.com/apache/carbondata/blob/master/docs/datamap/timeseries-datamap-guide.md) | Automatic |
+| mv | multi-table pre-aggregate table, | No DMPROPERTY is required | Manual |
+| lucene | lucene indexing for text column | index_columns to specifying the index columns | Manual/Automatic |
+| bloom | bloom filter for high cardinality column, geospatial column | index_columns to specifying the index columns | Manual/Automatic |
+
+## DataMap Management
+
+There are two kinds of management semantic for DataMap.
+
+1. Autmatic Refresh: Create datamap without `WITH DEFERED REBUILD` in the statement
+2. Manual Refresh: Create datamap with `WITH DEFERED REBUILD` in the statement
+
+### Automatic Refresh
+
+When user creates a datamap on the main table without using `WITH DEFERED REBUILD` syntax, the datamap will be managed by system automatically.
+For every data load to the main table, system will immediately triger a load to the datamap automatically. These two data loading (to main table and datamap) is executed in a transactional manner, meaning that it will be either both success or neither success.
+
+The data loading to datamap is incremental based on Segment concept, avoiding a expesive total rebuild.
+
+If user perform following command on the main table, system will return failure. (reject the operation)
+
+1. Data management command: `UPDATE/DELETE/DELETE SEGMENT`.
+2. Schema management command: `ALTER TABLE DROP COLUMN`, `ALTER TABLE CHANGE DATATYPE`,
+ `ALTER TABLE RENAME`. Note that adding a new column is supported, and for dropping columns and
+ change datatype command, CarbonData will check whether it will impact the pre-aggregate table, if
+ not, the operation is allowed, otherwise operation will be rejected by throwing exception.
+3. Partition management command: `ALTER TABLE ADD/DROP PARTITION
+
+If user do want to perform above operations on the main table, user can first drop the datamap, perform the operation, and re-create the datamap again.
+
+If user drop the main table, the datamap will be dropped immediately too.
+
+### Manual Refresh
+
+When user creates a datamap specifying maunal refresh semantic, the datamap is created with status *disabled* and query will NOT use this datamap until user can issue REBUILD DATAMAP command to build the datamap. For every REBUILD DATAMAP command, system will trigger a full rebuild of the datamap. After rebuild is done, system will change datamap status to *enabled*, so that it can be used in query rewrite.
+
+For every new data loading, data update, delete, the related datamap will be made *disabled*.
+
+If the main table is dropped by user, the related datamap will be dropped immediately.
+
+*Note: If you are creating a datamap on external table, you need to do manual managment of the datamap.*
+
+
+
+## DataMap Catalog
+
+Currently, when user creates a datamap, system will store the datamap metadata in a configurable *system* folder in HDFS or S3.
+
+In this *system* folder, it contains:
+
+- DataMapSchema file. It is a json file containing schema for one datamap. Ses DataMapSchema class. If user creates 100 datamaps (on different tables), there will be 100 files in *system* folder.
+- DataMapStatus file. Only one file, it is in json format, and each entry in the file represents for one datamap. Ses DataMapStatusDetail class
+
+There is a DataMapCatalog interface to retrieve schema of all datamap, it can be used in optimizer to get the metadata of datamap.
+
+
+
+## DataMap Related Commands
+
+### Explain
+
+How can user know whether datamap is used in the query?
+
+User can use EXPLAIN command to know, it will print out something like
+
+```text
+== CarbonData Profiler ==
+Hit mv DataMap: datamap1
+Scan Table: default.datamap1_table
++- filter:
++- pruning by CG DataMap
++- all blocklets: 1
+ skipped blocklets: 0
+```
+
+### Show DataMap
+
+There is a SHOW DATAMAPS command, when this is issued, system will read all datamap from *system* folder and print all information on screen. The current information includes:
+
+- DataMapName
+- DataMapProviderName like mv, preaggreagte, timeseries, etc
+- Associated Table
+
+### Compaction on DataMap
+
+This feature applies for preaggregate datamap only
+
+Running Compaction command (`ALTER TABLE COMPACT`) on main table will **not automatically** compact the pre-aggregate tables created on the main table. User need to run Compaction command separately on each pre-aggregate table to compact them.
+
+Compaction is an optional operation for pre-aggregate table. If compaction is performed on main table but not performed on pre-aggregate table, all queries still can benefit from pre-aggregate tables. To further improve the query performance, compaction on pre-aggregate tables can be triggered to merge the segments and files in the pre-aggregate tables.
[25/50] [abbrv] carbondata git commit: [CARBONDATA-2599] Use
RowStreamParserImp as default value of config 'carbon.stream.parser'
Posted by ja...@apache.org.
[CARBONDATA-2599] Use RowStreamParserImp as default value of config 'carbon.stream.parser'
Parser 'RowStreamParserImpl' is used more often for real scene, so use 'RowStreamParserImpl' as default value of config 'carbon.stream.parser'
This closes #2370
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f1163524
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f1163524
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f1163524
Branch: refs/heads/carbonstore
Commit: f1163524f5adab2dfeab992e17d6aac2b5bacf47
Parents: efad40d
Author: Zhang Zhichao <44...@qq.com>
Authored: Tue Jun 12 23:55:57 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Jun 15 01:38:27 2018 +0800
----------------------------------------------------------------------
docs/streaming-guide.md | 11 ++++++-----
.../carbondata/examples/SparkStreamingExample.scala | 3 ---
.../examples/StreamingWithRowParserExample.scala | 3 ---
.../carbondata/examples/StructuredStreamingExample.scala | 3 +++
.../spark/carbondata/TestStreamingTableOperation.scala | 5 +++++
.../carbondata/TestStreamingTableWithRowParser.scala | 3 ---
.../carbondata/streaming/parser/CarbonStreamParser.java | 4 +++-
7 files changed, 17 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1163524/docs/streaming-guide.md
----------------------------------------------------------------------
diff --git a/docs/streaming-guide.md b/docs/streaming-guide.md
index a9b174f..a9284e6 100644
--- a/docs/streaming-guide.md
+++ b/docs/streaming-guide.md
@@ -28,6 +28,7 @@ Start spark-shell in new terminal, type :paste, then copy and run the following
import org.apache.spark.sql.CarbonSession._
import org.apache.spark.sql.streaming.{ProcessingTime, StreamingQuery}
import org.apache.carbondata.core.util.path.CarbonTablePath
+ import org.apache.carbondata.streaming.parser.CarbonStreamParser
val warehouse = new File("./warehouse").getCanonicalPath
val metastore = new File("./metastore").getCanonicalPath
@@ -71,6 +72,8 @@ Start spark-shell in new terminal, type :paste, then copy and run the following
.option("checkpointLocation", CarbonTablePath.getStreamingCheckpointDir(tablePath))
.option("dbName", "default")
.option("tableName", "carbon_table")
+ .option(CarbonStreamParser.CARBON_STREAM_PARSER,
+ CarbonStreamParser.CARBON_STREAM_PARSER_CSV)
.start()
// start new thread to show data
@@ -157,13 +160,13 @@ Config the property "carbon.stream.parser" to define a stream parser to convert
property name | default | description
--- | --- | ---
-carbon.stream.parser | org.apache.carbondata.streaming.parser.CSVStreamParserImp | the class of the stream parser
+carbon.stream.parser | org.apache.carbondata.streaming.parser.RowStreamParserImp | the class of the stream parser
Currently CarbonData support two parsers, as following:
-**1. org.apache.carbondata.streaming.parser.CSVStreamParserImp**: This is the default stream parser, it gets a line data(String type) from the first index of InternalRow and converts this String to Object[].
+**1. org.apache.carbondata.streaming.parser.CSVStreamParserImp**: This parser gets a line data(String type) from the first index of InternalRow and converts this String to Object[].
-**2. org.apache.carbondata.streaming.parser.RowStreamParserImp**: This stream parser will auto convert InternalRow to Object[] according to schema of this `DataSet`, for example:
+**2. org.apache.carbondata.streaming.parser.RowStreamParserImp**: This is the default stream parser, it will auto convert InternalRow to Object[] according to schema of this `DataSet`, for example:
```scala
case class FileElement(school: Array[String], age: Int)
@@ -191,8 +194,6 @@ Currently CarbonData support two parsers, as following:
.option("checkpointLocation", tablePath.getStreamingCheckpointDir)
.option("dbName", "default")
.option("tableName", "carbon_table")
- .option(CarbonStreamParser.CARBON_STREAM_PARSER,
- CarbonStreamParser.CARBON_STREAM_PARSER_ROW_PARSER)
.start()
...
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1163524/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkStreamingExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkStreamingExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkStreamingExample.scala
index 27ea893..beaeee1 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkStreamingExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkStreamingExample.scala
@@ -30,7 +30,6 @@ import org.apache.spark.streaming.{Seconds, StreamingContext, Time}
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.examples.util.ExampleUtils
import org.apache.carbondata.streaming.CarbonSparkStreamingListener
-import org.apache.carbondata.streaming.parser.CarbonStreamParser
/**
* This example introduces how to use Spark Streaming to write data
@@ -172,8 +171,6 @@ object SparkStreamingExample {
" at batch time: " + time.toString() +
" the count of received data: " + df.count())
CarbonSparkStreamingFactory.getStreamSparkStreamingWriter(spark, "default", tableName)
- .option(CarbonStreamParser.CARBON_STREAM_PARSER,
- CarbonStreamParser.CARBON_STREAM_PARSER_ROW_PARSER)
.mode(SaveMode.Append)
.writeStreamData(df, time)
}}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1163524/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamingWithRowParserExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamingWithRowParserExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamingWithRowParserExample.scala
index 109629e..ceb3d0f 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamingWithRowParserExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamingWithRowParserExample.scala
@@ -25,7 +25,6 @@ import org.apache.spark.sql.streaming.{ProcessingTime, StreamingQuery}
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.examples.util.ExampleUtils
-import org.apache.carbondata.streaming.parser.CarbonStreamParser
case class FileElement(school: Array[String], age: Int)
case class StreamData(id: Int, name: String, city: String, salary: Float, file: FileElement)
@@ -170,8 +169,6 @@ object StreamingWithRowParserExample {
.option("checkpointLocation", CarbonTablePath.getStreamingCheckpointDir(tablePath))
.option("dbName", "default")
.option("tableName", "stream_table_with_row_parser")
- .option(CarbonStreamParser.CARBON_STREAM_PARSER,
- CarbonStreamParser.CARBON_STREAM_PARSER_ROW_PARSER)
.start()
qry.awaitTermination()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1163524/examples/spark2/src/main/scala/org/apache/carbondata/examples/StructuredStreamingExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/StructuredStreamingExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/StructuredStreamingExample.scala
index 38a1941..f88d8ee 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/StructuredStreamingExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/StructuredStreamingExample.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.streaming.{ProcessingTime, StreamingQuery}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.examples.util.ExampleUtils
+import org.apache.carbondata.streaming.parser.CarbonStreamParser
// scalastyle:off println
object StructuredStreamingExample {
@@ -156,6 +157,8 @@ object StructuredStreamingExample {
CarbonTablePath.getStreamingCheckpointDir(carbonTable.getTablePath))
.option("dbName", "default")
.option("tableName", "stream_table")
+ .option(CarbonStreamParser.CARBON_STREAM_PARSER,
+ CarbonStreamParser.CARBON_STREAM_PARSER_CSV)
.start()
qry.awaitTermination()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1163524/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index 325722d..3253c3d 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -40,6 +40,7 @@ import org.apache.carbondata.core.statusmanager.{FileFormat, SegmentStatus}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.spark.exception.ProcessMetaDataException
+import org.apache.carbondata.streaming.parser.CarbonStreamParser
class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
@@ -1713,6 +1714,8 @@ sql("drop table if exists streaming.bad_record_ignore")
.option("BAD_RECORD_PATH", badRecordsPath)
.option("dbName", tableIdentifier.database.get)
.option("tableName", tableIdentifier.table)
+ .option(CarbonStreamParser.CARBON_STREAM_PARSER,
+ CarbonStreamParser.CARBON_STREAM_PARSER_CSV)
.option(CarbonCommonConstants.HANDOFF_SIZE, handoffSize)
.option("timestampformat", CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
.option(CarbonCommonConstants.ENABLE_AUTO_HANDOFF, autoHandoff)
@@ -1830,6 +1833,8 @@ sql("drop table if exists streaming.bad_record_ignore")
.option("dbName", tableIdentifier.database.get)
.option("tableName", tableIdentifier.table)
.option("timestampformat", CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ .option(CarbonStreamParser.CARBON_STREAM_PARSER,
+ CarbonStreamParser.CARBON_STREAM_PARSER_CSV)
.start()
qry.awaitTermination()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1163524/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
index a6b0fec..39d63bf 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
@@ -35,7 +35,6 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.statusmanager.{FileFormat, SegmentStatus}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.streaming.parser.CarbonStreamParser
case class FileElement(school: Array[String], age: Integer)
case class StreamData(id: Integer, name: String, city: String, salary: java.lang.Float,
@@ -783,8 +782,6 @@ class TestStreamingTableWithRowParser extends QueryTest with BeforeAndAfterAll {
.option(CarbonCommonConstants.HANDOFF_SIZE, handoffSize)
.option("timestampformat", CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
.option(CarbonCommonConstants.ENABLE_AUTO_HANDOFF, autoHandoff)
- .option(CarbonStreamParser.CARBON_STREAM_PARSER,
- CarbonStreamParser.CARBON_STREAM_PARSER_ROW_PARSER)
.start()
qry.awaitTermination()
} catch {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f1163524/streaming/src/main/java/org/apache/carbondata/streaming/parser/CarbonStreamParser.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/carbondata/streaming/parser/CarbonStreamParser.java b/streaming/src/main/java/org/apache/carbondata/streaming/parser/CarbonStreamParser.java
index e335626..94f0307 100644
--- a/streaming/src/main/java/org/apache/carbondata/streaming/parser/CarbonStreamParser.java
+++ b/streaming/src/main/java/org/apache/carbondata/streaming/parser/CarbonStreamParser.java
@@ -28,12 +28,14 @@ public interface CarbonStreamParser {
String CARBON_STREAM_PARSER = "carbon.stream.parser";
- String CARBON_STREAM_PARSER_DEFAULT =
+ String CARBON_STREAM_PARSER_CSV =
"org.apache.carbondata.streaming.parser.CSVStreamParserImp";
String CARBON_STREAM_PARSER_ROW_PARSER =
"org.apache.carbondata.streaming.parser.RowStreamParserImp";
+ String CARBON_STREAM_PARSER_DEFAULT = CARBON_STREAM_PARSER_ROW_PARSER;
+
void initialize(Configuration configuration, StructType structType);
Object[] parserRow(InternalRow value);
[34/50] [abbrv] carbondata git commit: [CARBONDATA-2418] [Presto]
[S3] Fixed Presto Can't Query CarbonData When CarbonStore is at S3
Posted by ja...@apache.org.
[CARBONDATA-2418] [Presto] [S3] Fixed Presto Can't Query CarbonData When CarbonStore is at S3
This closes #2287
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dc4f87ba
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dc4f87ba
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dc4f87ba
Branch: refs/heads/carbonstore
Commit: dc4f87ba5047568e3800c6ed873846b408131da0
Parents: e7fed36
Author: anubhav100 <an...@knoldus.in>
Authored: Tue May 1 09:59:05 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Tue Jun 19 00:25:33 2018 +0800
----------------------------------------------------------------------
integration/presto/README.md | 28 +++++++
integration/presto/pom.xml | 49 +++++++++++-
.../carbondata/presto/CarbondataMetadata.java | 6 ++
.../presto/impl/CarbonTableConfig.java | 81 +++++++++++++++++++-
.../presto/impl/CarbonTableReader.java | 23 +++++-
5 files changed, 183 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc4f87ba/integration/presto/README.md
----------------------------------------------------------------------
diff --git a/integration/presto/README.md b/integration/presto/README.md
index 8da70d7..53884a2 100644
--- a/integration/presto/README.md
+++ b/integration/presto/README.md
@@ -82,6 +82,34 @@ Please follow the below steps to query carbondata in presto
For example, if you have a schema named 'default' stored in hdfs://namenode:9000/test/carbondata/,
Then set carbondata-store=hdfs://namenode:9000/test/carbondata
+#### Connecting to carbondata store on s3
+ * In case you want to query carbonstore on S3 using S3A api put following additional properties inside $PRESTO_HOME$/etc/catalog/carbondata.properties
+ ```
+ Required properties
+
+ fs.s3a.access.key={value}
+ fs.s3a.secret.key={value}
+
+ Optional properties
+
+ fs.s3a.endpoint={value}
+ ```
+ * In case you want to query carbonstore on s3 using S3 api put following additional properties inside $PRESTO_HOME$/etc/catalog/carbondata.properties
+ ```
+ fs.s3.awsAccessKeyId={value}
+ fs.s3.awsSecretAccessKey={value}
+ ```
+ * In case You want to query carbonstore on s3 using S3N api put following additional properties inside $PRESTO_HOME$/etc/catalog/carbondata.properties
+ ```
+ fs.s3n.awsAccessKeyId={value}
+ fs.s3n.awsSecretAccessKey={value}
+ ```
+
+ Replace the schema-store-path with the absolute path of the parent directory of the schema.
+ For example, if you have a schema named 'default' stored in a bucket s3a://s3-carbon/store,
+ Then set carbondata-store=s3a://s3-carbon/store
+
+#### Unsafe Properties
enable.unsafe.in.query.processing property by default is true in CarbonData system, the carbon.unsafe.working.memory.in.mb
property defines the limit for Unsafe Memory usage in Mega Bytes, the default value is 512 MB.
If your tables are big you can increase the unsafe memory, or disable unsafe via setting enable.unsafe.in.query.processing=false.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc4f87ba/integration/presto/pom.xml
----------------------------------------------------------------------
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index bfa05f9..b91f070 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -498,7 +498,54 @@
<artifactId>lz4-java</artifactId>
<version>1.4.0</version>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>1.7.4</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ <version>0.9.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>4.2</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc4f87ba/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
index 718c628..8be7494 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
@@ -17,8 +17,11 @@
package org.apache.carbondata.presto;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
+import org.apache.carbondata.presto.impl.CarbonTableConfig;
import org.apache.carbondata.presto.impl.CarbonTableReader;
import com.facebook.presto.spi.*;
import com.facebook.presto.spi.connector.ConnectorMetadata;
@@ -37,6 +40,9 @@ import java.util.*;
import static org.apache.carbondata.presto.Types.checkType;
import static com.google.common.base.Preconditions.checkArgument;
import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
+import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY;
public class CarbondataMetadata implements ConnectorMetadata {
private final String connectorId;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc4f87ba/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
index 6a96221..75a7f11 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
@@ -17,10 +17,10 @@
package org.apache.carbondata.presto.impl;
-import io.airlift.configuration.Config;
-
import javax.validation.constraints.NotNull;
+import io.airlift.configuration.Config;
+
/**
* Configuration read from etc/catalog/carbondata.properties
*/
@@ -32,6 +32,14 @@ public class CarbonTableConfig {
private String storePath;
private String unsafeMemoryInMb;
private String enableUnsafeInQueryExecution;
+ private String s3A_acesssKey;
+ private String s3A_secretKey;
+ private String s3_acesssKey;
+ private String s3_secretKey;
+ private String s3N_acesssKey;
+ private String s3N_secretKey;
+ private String endPoint;
+
@NotNull public String getDbPath() {
return dbPath;
@@ -79,4 +87,73 @@ public class CarbonTableConfig {
this.enableUnsafeInQueryExecution = enableUnsafeInQueryExecution;
return this;
}
+
+ public String getS3A_AcesssKey() {
+ return s3A_acesssKey;
+ }
+
+ public String getS3A_SecretKey() {
+ return s3A_secretKey;
+ }
+
+ public String getS3_AcesssKey() {
+ return s3_acesssKey;
+ }
+
+ public String getS3_SecretKey() {
+ return s3_secretKey;
+ }
+
+ public String getS3N_AcesssKey() {
+ return s3N_acesssKey;
+ }
+
+ public String getS3N_SecretKey() {
+ return s3N_secretKey;
+ }
+
+ public String getS3EndPoint() {
+ return endPoint;
+ }
+
+
+ @Config("fs.s3a.access.key")
+ public CarbonTableConfig setS3A_AcesssKey(String s3A_acesssKey) {
+ this.s3A_acesssKey = s3A_acesssKey;
+ return this;
+ }
+
+ @Config("fs.s3a.secret.key")
+ public CarbonTableConfig setS3A_SecretKey(String s3A_secretKey) {
+ this.s3A_secretKey = s3A_secretKey;
+ return this;
+ }
+
+ @Config("fs.s3.awsAccessKeyId")
+ public CarbonTableConfig setS3_AcesssKey(String s3_acesssKey) {
+ this.s3_acesssKey = s3_acesssKey;
+ return this;
+ }
+
+ @Config("fs.s3.awsSecretAccessKey")
+ public CarbonTableConfig setS3_SecretKey(String s3_secretKey) {
+ this.s3_secretKey = s3_secretKey;
+ return this;
+ }
+ @Config("fs.s3n.awsAccessKeyId")
+ public CarbonTableConfig setS3N_AcesssKey(String s3N_acesssKey) {
+ this.s3N_acesssKey = s3N_acesssKey;
+ return this;
+ }
+
+ @Config("fs.s3.awsSecretAccessKey")
+ public CarbonTableConfig setS3N_SecretKey(String s3N_secretKey) {
+ this.s3N_secretKey = s3N_secretKey;
+ return this;
+ }
+ @Config("fs.s3a.endpoint")
+ public CarbonTableConfig setS3EndPoint(String endPoint) {
+ this.endPoint = endPoint;
+ return this;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc4f87ba/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
index ad5b7ee..5866ad1 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
@@ -25,6 +25,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;
@@ -79,6 +80,9 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.thrift.TBase;
import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
+import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY;
/**
* CarbonTableReader will be a facade of these utils
@@ -98,7 +102,7 @@ public class CarbonTableReader {
return CarbonTablePath.isCarbonDataFile(path.getName());
}
};
- private CarbonTableConfig config;
+ public CarbonTableConfig config;
/**
* The names of the tables under the schema (this.carbonFileList).
*/
@@ -132,6 +136,7 @@ public class CarbonTableReader {
this.config = requireNonNull(config, "CarbonTableConfig is null");
this.carbonCache = new AtomicReference(new HashMap());
tableList = new ConcurrentSet<>();
+ setS3Properties();
}
/**
@@ -506,5 +511,21 @@ public class CarbonTableReader {
return format;
}
+ private void setS3Properties(){
+ FileFactory.getConfiguration()
+ .set(ACCESS_KEY, Objects.toString(config.getS3A_AcesssKey(),""));
+ FileFactory.getConfiguration()
+ .set(SECRET_KEY, Objects.toString(config.getS3A_SecretKey()));
+ FileFactory.getConfiguration().set(CarbonCommonConstants.S3_ACCESS_KEY,
+ Objects.toString(config.getS3_AcesssKey(),""));
+ FileFactory.getConfiguration().set(CarbonCommonConstants.S3_SECRET_KEY,
+ Objects.toString(config.getS3_SecretKey()));
+ FileFactory.getConfiguration().set(CarbonCommonConstants.S3N_ACCESS_KEY,
+ Objects.toString(config.getS3N_AcesssKey(),""));
+ FileFactory.getConfiguration().set(CarbonCommonConstants.S3N_SECRET_KEY,
+ Objects.toString(config.getS3N_SecretKey(),""));
+ FileFactory.getConfiguration().set(ENDPOINT,
+ Objects.toString(config.getS3EndPoint(),""));
+}
}
\ No newline at end of file
[13/50] [abbrv] carbondata git commit: [CARBONDATA-2577]
[CARBONDATA-2579] Fixed issue in Avro logical type for nested Array and
document update
Posted by ja...@apache.org.
[CARBONDATA-2577] [CARBONDATA-2579] Fixed issue in Avro logical type for nested Array and document update
Problem: Nested Array logical type of date, timestamp-millis, timestamp-micros is not working.
Root cause: During the preparation of carbon schema from avro schema. For array nested type logical types were not handled.
Solution: Handle the logical types for array nested type during carbon schema preparation.
This closes #2361
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/041603dc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/041603dc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/041603dc
Branch: refs/heads/carbonstore
Commit: 041603dccf1d98348db36c4bf8e2e60d50a5bcc8
Parents: d401e06
Author: ajantha-bhat <aj...@gmail.com>
Authored: Mon Jun 4 16:12:48 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Tue Jun 5 19:21:07 2018 +0530
----------------------------------------------------------------------
docs/data-management-on-carbondata.md | 7 ++++-
docs/sdk-guide.md | 15 +++++++++++
.../TestNonTransactionalCarbonTable.scala | 28 +++++++++-----------
.../carbondata/sdk/file/AvroCarbonWriter.java | 26 ++++++++++++++++--
4 files changed, 58 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/041603dc/docs/data-management-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/data-management-on-carbondata.md b/docs/data-management-on-carbondata.md
index 706209c..3326e9b 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -216,7 +216,12 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
This can be SDK output. Refer [SDK Writer Guide](https://github.com/apache/carbondata/blob/master/docs/sdk-writer-guide.md).
**Note:**
- Dropping of the external table should not delete the files present in the location.
+ 1. Dropping of the external table should not delete the files present in the location.
+ 2. When external table is created on non-transactional table data,
+ external table will be registered with the schema of carbondata files.
+ If multiple files with different schema is present, exception will be thrown.
+ So, If table registered with one schema and files are of different schema,
+ suggest to drop the external table and create again to register table with new schema.
## CREATE DATABASE
http://git-wip-us.apache.org/repos/asf/carbondata/blob/041603dc/docs/sdk-guide.md
----------------------------------------------------------------------
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index 0f20dc3..e04698d 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -128,6 +128,21 @@ Each of SQL data types are mapped into data types of SDK. Following are the mapp
| STRING | DataTypes.STRING |
| DECIMAL | DataTypes.createDecimalType(precision, scale) |
+**NOTE:**
+ Carbon Supports below logical types of AVRO.
+ a. Date
+ The date logical type represents a date within the calendar, with no reference to a particular time zone or time of day.
+ A date logical type annotates an Avro int, where the int stores the number of days from the unix epoch, 1 January 1970 (ISO calendar).
+ b. Timestamp (millisecond precision)
+ The timestamp-millis logical type represents an instant on the global timeline, independent of a particular time zone or calendar, with a precision of one millisecond.
+ A timestamp-millis logical type annotates an Avro long, where the long stores the number of milliseconds from the unix epoch, 1 January 1970 00:00:00.000 UTC.
+ c. Timestamp (microsecond precision)
+ The timestamp-micros logical type represents an instant on the global timeline, independent of a particular time zone or calendar, with a precision of one microsecond.
+ A timestamp-micros logical type annotates an Avro long, where the long stores the number of microseconds from the unix epoch, 1 January 1970 00:00:00.000000 UTC.
+
+ Currently the values of logical types are not validated by carbon.
+ Expect that avro record passed by the user is already validated by avro record generator tools.
+
## Run SQL on files directly
Instead of creating table and query it, you can also query that file directly with SQL.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/041603dc/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index 14a63ca..b275bb8 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -1825,6 +1825,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
| "items": {
| "name": "EachdoorNums",
| "type": "int",
+ | "logicalType": "date",
| "default": -1
| }
| }
@@ -1849,8 +1850,8 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
buildAvroTestDataMultiLevel3_2(3, null)
}
- // test multi level -- 3 levels [array of array of array of int]
- test("test multi level support : array of array of array of int") {
+ // test multi level -- 3 levels [array of array of array of int with logical type]
+ test("test multi level support : array of array of array of int with logical type") {
buildAvroTestDataMultiLevel3_2Type()
assert(new File(writerPath).exists())
sql("DROP TABLE IF EXISTS sdkOutputTable")
@@ -1858,22 +1859,19 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY 'carbondata' LOCATION
|'$writerPath' """.stripMargin)
- sql("select * from sdkOutputTable").show(false)
+ sql("select * from sdkOutputTable limit 1").show(false)
// TODO: Add a validation
/*
- +----+---+---------------------------------------------------------------------------+
- |name|age|BuildNum
- |
- +----+---+---------------------------------------------------------------------------+
- |bob |10 |[WrappedArray(WrappedArray(1, 2, 3), WrappedArray(4, 5, 6)), WrappedArray
- (WrappedArray(10, 20, 30), WrappedArray(40, 50, 60))]|
- |bob |10 |[WrappedArray(WrappedArray(1, 2, 3), WrappedArray(4, 5, 6)), WrappedArray
- (WrappedArray(10, 20, 30), WrappedArray(40, 50, 60))]|
- |bob |10 |[WrappedArray(WrappedArray(1, 2, 3), WrappedArray(4, 5, 6)), WrappedArray
- (WrappedArray(10, 20, 30), WrappedArray(40, 50, 60))]|
- +----+---+---------------------------------------------------------------------------+
- */
+ +----+---+------------------------------------------------------------------+
+ |name|age|BuildNum |
+ +----+---+------------------------------------------------------------------+
+ |bob |10 |[WrappedArray(WrappedArray(1970-01-02, 1970-01-03, 1970-01-04), |
+ | WrappedArray(1970-01-05, 1970-01-06, 1970-01-07)), |
+ | WrappedArray(WrappedArray(1970-01-11, 1970-01-21, 1970-01-31), |
+ | WrappedArray(1970-02-10, 1970-02-20, 1970-03-02))] |
+ +----+---+------------------------------------------------------------------+
+ */
sql("DROP TABLE sdkOutputTable")
// drop table should not delete the files
http://git-wip-us.apache.org/repos/asf/carbondata/blob/041603dc/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
index edecd6b..fdd1f5a 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
@@ -323,13 +323,35 @@ public class AvroCarbonWriter extends CarbonWriter {
}
private static DataType getMappingDataTypeForArrayRecord(Schema childSchema) {
+ LogicalType logicalType = childSchema.getLogicalType();
switch (childSchema.getType()) {
case BOOLEAN:
return DataTypes.BOOLEAN;
case INT:
- return DataTypes.INT;
+ if (logicalType != null) {
+ if (logicalType instanceof LogicalTypes.Date) {
+ return DataTypes.DATE;
+ } else {
+ LOGGER.warn("Unsupported logical type. Considering Data Type as INT for " + childSchema
+ .getName());
+ return DataTypes.INT;
+ }
+ } else {
+ return DataTypes.INT;
+ }
case LONG:
- return DataTypes.LONG;
+ if (logicalType != null) {
+ if (logicalType instanceof LogicalTypes.TimestampMillis
+ || logicalType instanceof LogicalTypes.TimestampMicros) {
+ return DataTypes.TIMESTAMP;
+ } else {
+ LOGGER.warn("Unsupported logical type. Considering Data Type as LONG for " + childSchema
+ .getName());
+ return DataTypes.LONG;
+ }
+ } else {
+ return DataTypes.LONG;
+ }
case DOUBLE:
return DataTypes.DOUBLE;
case STRING:
[02/50] [abbrv] carbondata git commit: [CARBONDATA-2508] Fix the
exception that can't get executorService when start search mode twice
Posted by ja...@apache.org.
[CARBONDATA-2508] Fix the exception that can't get executorService when start search mode twice
This closes #2355
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6aadfe70
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6aadfe70
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6aadfe70
Branch: refs/heads/carbonstore
Commit: 6aadfe70a67bfd13ed5efedfaa368de57403a88f
Parents: 5b2b913
Author: xubo245 <xu...@huawei.com>
Authored: Thu May 31 09:15:16 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Thu May 31 14:24:35 2018 +0800
----------------------------------------------------------------------
.../executor/impl/SearchModeDetailQueryExecutor.java | 6 ++++--
.../impl/SearchModeVectorDetailQueryExecutor.java | 6 ++++--
.../testsuite/detailquery/SearchModeTestCase.scala | 13 +++++++++++++
3 files changed, 21 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6aadfe70/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java
index aed472c..ae14327 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java
@@ -37,8 +37,10 @@ public class SearchModeDetailQueryExecutor extends AbstractQueryExecutor<Object>
LogServiceFactory.getLogService(SearchModeDetailQueryExecutor.class.getName());
private static ExecutorService executorService = null;
- static {
- initThreadPool();
+ public SearchModeDetailQueryExecutor() {
+ if (executorService == null) {
+ initThreadPool();
+ }
}
private static synchronized void initThreadPool() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6aadfe70/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java
index 00fd511..705c451 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java
@@ -40,8 +40,10 @@ public class SearchModeVectorDetailQueryExecutor extends AbstractQueryExecutor<O
LogServiceFactory.getLogService(SearchModeVectorDetailQueryExecutor.class.getName());
private static ExecutorService executorService = null;
- static {
- initThreadPool();
+ public SearchModeVectorDetailQueryExecutor() {
+ if (executorService == null) {
+ initThreadPool();
+ }
}
private static synchronized void initThreadPool() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6aadfe70/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
index d278fc5..3e6adaf 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
@@ -136,4 +136,17 @@ class SearchModeTestCase extends QueryTest with BeforeAndAfterAll {
sql("DROP DATAMAP if exists dm3 ON TABLE main")
}
+ test("start search mode twice") {
+ sqlContext.sparkSession.asInstanceOf[CarbonSession].startSearchMode()
+ assert(sqlContext.sparkSession.asInstanceOf[CarbonSession].isSearchModeEnabled)
+ checkSearchAnswer("select id from main where id = '3' limit 10")
+ sqlContext.sparkSession.asInstanceOf[CarbonSession].stopSearchMode()
+ assert(!sqlContext.sparkSession.asInstanceOf[CarbonSession].isSearchModeEnabled)
+
+ // start twice
+ sqlContext.sparkSession.asInstanceOf[CarbonSession].startSearchMode()
+ assert(sqlContext.sparkSession.asInstanceOf[CarbonSession].isSearchModeEnabled)
+ checkSearchAnswer("select id from main where id = '3' limit 10")
+ sqlContext.sparkSession.asInstanceOf[CarbonSession].stopSearchMode()
+ }
}
[18/50] [abbrv] carbondata git commit: [CARBONDATA-2573] integrate
carbonstore mv branch
Posted by ja...@apache.org.
[CARBONDATA-2573] integrate carbonstore mv branch
Fixes bugs related to MV and added tests
This closes #2335
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0ef7e55c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0ef7e55c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0ef7e55c
Branch: refs/heads/carbonstore
Commit: 0ef7e55c46be9d3767539d1a51b780064cc7ad26
Parents: 83ee2c4
Author: ravipesala <ra...@gmail.com>
Authored: Wed May 30 09:11:13 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Mon Jun 11 21:25:31 2018 +0800
----------------------------------------------------------------------
.../carbondata/mv/datamap/MVAnalyzerRule.scala | 2 +-
.../apache/carbondata/mv/datamap/MVHelper.scala | 23 +
.../apache/carbondata/mv/datamap/MVState.scala | 55 --
.../mv/rewrite/DefaultMatchMaker.scala | 34 +-
.../carbondata/mv/rewrite/Navigator.scala | 50 +-
.../carbondata/mv/rewrite/QueryRewrite.scala | 19 +-
.../mv/rewrite/SummaryDatasetCatalog.scala | 79 +-
.../apache/carbondata/mv/rewrite/Utils.scala | 108 ++-
.../carbondata/mv/session/MVSession.scala | 84 ++
.../mv/session/internal/SessionState.scala | 56 ++
.../mv/rewrite/MVCreateTestCase.scala | 46 +-
.../carbondata/mv/rewrite/MVTPCDSTestCase.scala | 2 +-
.../SelectSelectExactChildrenSuite.scala | 5 +-
.../carbondata/mv/rewrite/TestSQLSuite.scala | 99 +++
.../carbondata/mv/rewrite/Tpcds_1_4_Suite.scala | 84 ++
.../mv/rewrite/matching/TestSQLBatch.scala | 23 +-
.../rewrite/matching/TestTPCDS_1_4_Batch.scala | 886 +++++++++++++------
.../org/apache/carbondata/mv/dsl/package.scala | 4 +-
.../util/LogicalPlanSignatureGenerator.scala | 11 +-
.../carbondata/mv/plans/util/SQLBuilder.scala | 14 +-
.../mv/testutil/Tpcds_1_4_Tables.scala | 142 +--
.../org/apache/carbondata/mv/TestSQLBatch.scala | 584 ------------
.../mv/plans/ExtractJoinConditionsSuite.scala | 2 +-
.../carbondata/mv/plans/IsSPJGHSuite.scala | 3 +-
.../mv/plans/LogicalToModularPlanSuite.scala | 3 +-
.../carbondata/mv/plans/ModularToSQLSuite.scala | 232 +++--
.../carbondata/mv/plans/SignatureSuite.scala | 95 +-
.../mv/plans/Tpcds_1_4_BenchmarkSuite.scala | 86 ++
.../carbondata/mv/testutil/TestSQLBatch.scala | 584 ++++++++++++
.../carbondata/mv/testutil/TestSQLBatch2.scala | 138 +++
30 files changed, 2306 insertions(+), 1247 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala
index 4e93f15..483780f 100644
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala
@@ -65,7 +65,7 @@ class MVAnalyzerRule(sparkSession: SparkSession) extends Rule[LogicalPlan] {
val catalog = DataMapStoreManager.getInstance().getDataMapCatalog(dataMapProvider,
DataMapClassProvider.MV.getShortName).asInstanceOf[SummaryDatasetCatalog]
if (needAnalysis && catalog != null && isValidPlan(plan, catalog)) {
- val modularPlan = catalog.mVState.rewritePlan(plan).withSummaryData
+ val modularPlan = catalog.mvSession.sessionState.rewritePlan(plan).withMVTable
if (modularPlan.find (_.rewritten).isDefined) {
val compactSQL = modularPlan.asCompactSQL
LOGGER.audit(s"\n$compactSQL\n")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala
index 0f9362f..a40fa2c 100644
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala
@@ -373,5 +373,28 @@ object MVHelper {
case other => other
}
}
+
+ /**
+ * Rewrite the updated mv query with corresponding MV table.
+ */
+ def rewriteWithMVTable(rewrittenPlan: ModularPlan, rewrite: QueryRewrite): ModularPlan = {
+ if (rewrittenPlan.find(_.rewritten).isDefined) {
+ val updatedDataMapTablePlan = rewrittenPlan transform {
+ case s: Select =>
+ MVHelper.updateDataMap(s, rewrite)
+ case g: GroupBy =>
+ MVHelper.updateDataMap(g, rewrite)
+ }
+ // TODO Find a better way to set the rewritten flag, it may fail in some conditions.
+ val mapping =
+ rewrittenPlan.collect { case m: ModularPlan => m } zip
+ updatedDataMapTablePlan.collect { case m: ModularPlan => m }
+ mapping.foreach(f => if (f._1.rewritten) f._2.setRewritten())
+
+ updatedDataMapTablePlan
+ } else {
+ rewrittenPlan
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVState.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVState.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVState.scala
deleted file mode 100644
index 412d547..0000000
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVState.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.mv.datamap
-
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-
-import org.apache.carbondata.mv.plans.modular.SimpleModularizer
-import org.apache.carbondata.mv.plans.util.BirdcageOptimizer
-import org.apache.carbondata.mv.rewrite.{DefaultMatchMaker, Navigator, QueryRewrite, SummaryDatasetCatalog}
-
-/**
- * A class that holds all session-specific state.
- */
-private[mv] class MVState(summaryDatasetCatalog: SummaryDatasetCatalog) {
-
- // Note: These are all lazy vals because they depend on each other (e.g. conf) and we
- // want subclasses to override some of the fields. Otherwise, we would get a lot of NPEs.
-
- /**
- * Modular query plan modularizer
- */
- lazy val modularizer = SimpleModularizer
-
- /**
- * Logical query plan optimizer.
- */
- lazy val optimizer = BirdcageOptimizer
-
- lazy val matcher = DefaultMatchMaker
-
- lazy val navigator: Navigator = new Navigator(summaryDatasetCatalog, this)
-
- /**
- * Rewrite the logical query plan to MV plan if applicable.
- * @param plan
- * @return
- */
- def rewritePlan(plan: LogicalPlan): QueryRewrite = new QueryRewrite(this, plan)
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/DefaultMatchMaker.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/DefaultMatchMaker.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/DefaultMatchMaker.scala
index 899c36c..6dbf236 100644
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/DefaultMatchMaker.scala
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/DefaultMatchMaker.scala
@@ -15,6 +15,7 @@
* limitations under the License.
*/
+
package org.apache.carbondata.mv.rewrite
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, AttributeReference, AttributeSet, Expression, PredicateHelper, _}
@@ -444,23 +445,40 @@ object GroupbyGroupbySelectOnlyChildDelta extends DefaultMatchPattern with Predi
if (isGroupingEdR && ((!needRegrouping && isAggEmR) || needRegrouping) && canPullup) {
// pull up
val pullupOutputList = gb_2a.outputList.map(_.toAttribute) ++ rejoinOutputList
- val sel_2c1 = sel_1c1.copy(
- outputList = pullupOutputList,
- inputList = pullupOutputList,
- children = sel_1c1.children.map {
- case s: Select => gb_2a
- case other => other })
+ val myOutputList = gb_2a.outputList.filter {
+ case alias: Alias => gb_2q.outputList.filter(_.isInstanceOf[Alias])
+ .exists(_.asInstanceOf[Alias].child.semanticEquals(alias.child))
+ case attr: Attribute => gb_2q.outputList.exists(_.semanticEquals(attr))
+ }.map(_.toAttribute) ++ rejoinOutputList
+ // TODO: find out if we really need to check needRegrouping or just use myOutputList
+ val sel_2c1 = if (needRegrouping) {
+ sel_1c1
+ .copy(outputList = pullupOutputList,
+ inputList = pullupOutputList,
+ children = sel_1c1.children
+ .map { _ match { case s: modular.Select => gb_2a; case other => other } })
+ } else {
+ sel_1c1
+ .copy(outputList = myOutputList,
+ inputList = pullupOutputList,
+ children = sel_1c1.children
+ .map { _ match { case s: modular.Select => gb_2a; case other => other } })
+ }
+ // sel_1c1.copy(outputList = pullupOutputList, inputList = pullupOutputList, children =
+ // sel_1c1.children.map { _ match { case s: modular.Select => gb_2a; case other =>
+ // other } })
if (rejoinOutputList.isEmpty) {
val aliasMap = AttributeMap(gb_2a.outputList.collect {
- case a: Alias => (a.toAttribute, a) })
+ case a: Alias => (a.toAttribute, a)
+ })
Utils.tryMatch(gb_2a, gb_2q, aliasMap).flatMap {
case g: GroupBy => Some(g.copy(child = sel_2c1));
case _ => None
}.map { wip =>
factorOutSubsumer(wip, gb_2a, sel_1c1.aliasMap)
}.map(Seq(_))
- .getOrElse(Nil)
+ .getOrElse(Nil)
}
// TODO: implement regrouping with 1:N rejoin (rejoin tables being the "1" side)
// via catalog service
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/Navigator.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/Navigator.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/Navigator.scala
index 545920e..a36988a 100644
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/Navigator.scala
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/Navigator.scala
@@ -19,35 +19,38 @@ package org.apache.carbondata.mv.rewrite
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, AttributeSet}
-import org.apache.carbondata.mv.datamap.{MVHelper, MVState}
import org.apache.carbondata.mv.expressions.modular._
import org.apache.carbondata.mv.plans.modular.{GroupBy, ModularPlan, Select}
import org.apache.carbondata.mv.plans.modular
+import org.apache.carbondata.mv.session.MVSession
-private[mv] class Navigator(catalog: SummaryDatasetCatalog, session: MVState) {
+private[mv] class Navigator(catalog: SummaryDatasetCatalog, session: MVSession) {
def rewriteWithSummaryDatasets(plan: ModularPlan, rewrite: QueryRewrite): ModularPlan = {
val replaced = plan.transformAllExpressions {
case s: ModularSubquery =>
if (s.children.isEmpty) {
- ScalarModularSubquery(
- rewriteWithSummaryDatasetsCore(s.plan, rewrite), s.children, s.exprId)
+ rewriteWithSummaryDatasetsCore(s.plan, rewrite) match {
+ case Some(rewrittenPlan) => ScalarModularSubquery(rewrittenPlan, s.children, s.exprId)
+ case None => s
+ }
}
else throw new UnsupportedOperationException(s"Rewrite expression $s isn't supported")
case o => o
}
- rewriteWithSummaryDatasetsCore(replaced, rewrite)
+ rewriteWithSummaryDatasetsCore(replaced, rewrite).getOrElse(replaced)
}
- def rewriteWithSummaryDatasetsCore(plan: ModularPlan, rewrite: QueryRewrite): ModularPlan = {
+ def rewriteWithSummaryDatasetsCore(plan: ModularPlan,
+ rewrite: QueryRewrite): Option[ModularPlan] = {
val rewrittenPlan = plan transformDown {
case currentFragment =>
if (currentFragment.rewritten || !currentFragment.isSPJGH) currentFragment
else {
val compensation =
(for { dataset <- catalog.lookupFeasibleSummaryDatasets(currentFragment).toStream
- subsumer <- session.modularizer.modularize(
- session.optimizer.execute(dataset.plan)).map(_.harmonized)
+ subsumer <- session.sessionState.modularizer.modularize(
+ session.sessionState.optimizer.execute(dataset.plan)) // .map(_.harmonized)
subsumee <- unifySubsumee(currentFragment)
comp <- subsume(
unifySubsumer2(
@@ -61,25 +64,10 @@ private[mv] class Navigator(catalog: SummaryDatasetCatalog, session: MVState) {
compensation.map(_.setRewritten).getOrElse(currentFragment)
}
}
- // In case it is rewritten plan and the datamap table is not updated then update the datamap
- // table in plan.
- if (rewrittenPlan.find(_.rewritten).isDefined) {
- val updatedDataMapTablePlan = rewrittenPlan transform {
- case s: Select =>
- MVHelper.updateDataMap(s, rewrite)
- case g: GroupBy =>
- MVHelper.updateDataMap(g, rewrite)
- }
- // TODO Find a better way to set the rewritten flag, it may fail in some conditions.
- val mapping =
- rewrittenPlan.collect {case m: ModularPlan => m } zip
- updatedDataMapTablePlan.collect {case m: ModularPlan => m}
- mapping.foreach(f => if (f._1.rewritten) f._2.setRewritten())
-
- updatedDataMapTablePlan
-
+ if (rewrittenPlan.fastEquals(plan)) {
+ None
} else {
- rewrittenPlan
+ Some(rewrittenPlan)
}
}
@@ -92,7 +80,7 @@ private[mv] class Navigator(catalog: SummaryDatasetCatalog, session: MVState) {
case (Nil, Nil) => None
case (r, e) if r.forall(_.isInstanceOf[modular.LeafNode]) &&
e.forall(_.isInstanceOf[modular.LeafNode]) =>
- val iter = session.matcher.execute(subsumer, subsumee, None, rewrite)
+ val iter = session.sessionState.matcher.execute(subsumer, subsumee, None, rewrite)
if (iter.hasNext) Some(iter.next)
else None
@@ -100,16 +88,18 @@ private[mv] class Navigator(catalog: SummaryDatasetCatalog, session: MVState) {
val compensation = subsume(rchild, echild, rewrite)
val oiter = compensation.map {
case comp if comp.eq(rchild) =>
- session.matcher.execute(subsumer, subsumee, None, rewrite)
+ session.sessionState.matcher.execute(subsumer, subsumee, None, rewrite)
case _ =>
- session.matcher.execute(subsumer, subsumee, compensation, rewrite)
+ session.sessionState.matcher.execute(subsumer, subsumee, compensation, rewrite)
}
oiter.flatMap { case iter if iter.hasNext => Some(iter.next)
case _ => None }
case _ => None
}
- } else None
+ } else {
+ None
+ }
}
private def updateDatamap(rchild: ModularPlan, subsume: ModularPlan) = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/QueryRewrite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/QueryRewrite.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/QueryRewrite.scala
index 5039d66..88bc155 100644
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/QueryRewrite.scala
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/QueryRewrite.scala
@@ -21,31 +21,38 @@ import java.util.concurrent.atomic.AtomicLong
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.carbondata.mv.datamap.MVState
+import org.apache.carbondata.mv.datamap.MVHelper
import org.apache.carbondata.mv.plans.modular.ModularPlan
+import org.apache.carbondata.mv.session.MVSession
/**
* The primary workflow for rewriting relational queries using Spark libraries.
+ * Designed to allow easy access to the intermediate phases of query rewrite for developers.
+ *
+ * While this is not a public class, we should avoid changing the function names for the sake of
+ * changing them, because a lot of developers use the feature for debugging.
*/
class QueryRewrite private (
- state: MVState,
+ state: MVSession,
logical: LogicalPlan,
nextSubqueryId: AtomicLong) {
self =>
- def this(state: MVState, logical: LogicalPlan) =
+ def this(state: MVSession, logical: LogicalPlan) =
this(state, logical, new AtomicLong(0))
def newSubsumerName(): String = s"gen_subsumer_${nextSubqueryId.getAndIncrement()}"
lazy val optimizedPlan: LogicalPlan =
- state.optimizer.execute(logical)
+ state.sessionState.optimizer.execute(logical)
lazy val modularPlan: ModularPlan =
- state.modularizer.modularize(optimizedPlan).next().harmonized
+ state.sessionState.modularizer.modularize(optimizedPlan).next().harmonized
lazy val withSummaryData: ModularPlan =
- state.navigator.rewriteWithSummaryDatasets(modularPlan, self)
+ state.sessionState.navigator.rewriteWithSummaryDatasets(modularPlan, self)
+
+ lazy val withMVTable: ModularPlan = MVHelper.rewriteWithMVTable(withSummaryData, this)
lazy val toCompactSQL: String = withSummaryData.asCompactSQL
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/SummaryDatasetCatalog.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/SummaryDatasetCatalog.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/SummaryDatasetCatalog.scala
index c29c08f..3b5930f 100644
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/SummaryDatasetCatalog.scala
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/SummaryDatasetCatalog.scala
@@ -28,9 +28,10 @@ import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.carbondata.core.datamap.DataMapCatalog
import org.apache.carbondata.core.datamap.status.DataMapStatusManager
import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
-import org.apache.carbondata.mv.datamap.{MVHelper, MVState}
+import org.apache.carbondata.mv.datamap.MVHelper
import org.apache.carbondata.mv.plans.modular.{Flags, ModularPlan, ModularRelation, Select}
import org.apache.carbondata.mv.plans.util.Signature
+import org.apache.carbondata.mv.session.MVSession
/** Holds a summary logical plan */
private[mv] case class SummaryDataset(signature: Option[Signature],
@@ -44,7 +45,7 @@ private[mv] class SummaryDatasetCatalog(sparkSession: SparkSession)
@transient
private val summaryDatasets = new scala.collection.mutable.ArrayBuffer[SummaryDataset]
- val mVState = new MVState(this)
+ val mvSession = new MVSession(sparkSession, this)
@transient
private val registerLock = new ReentrantReadWriteLock
@@ -54,6 +55,7 @@ private[mv] class SummaryDatasetCatalog(sparkSession: SparkSession)
*/
lazy val parser = new CarbonSpark2SqlParser
+
/** Acquires a read lock on the catalog for the duration of `f`. */
private def readLock[A](f: => A): A = {
val lock = registerLock.readLock()
@@ -97,9 +99,9 @@ private[mv] class SummaryDatasetCatalog(sparkSession: SparkSession)
val updatedQuery = parser.addPreAggFunction(dataMapSchema.getCtasQuery)
val query = sparkSession.sql(updatedQuery)
val planToRegister = MVHelper.dropDummFuc(query.queryExecution.analyzed)
- val modularPlan = mVState.modularizer.modularize(mVState.optimizer.execute(planToRegister))
- .next()
- .harmonized
+ val modularPlan =
+ mvSession.sessionState.modularizer.modularize(
+ mvSession.sessionState.optimizer.execute(planToRegister)).next().harmonized
val signature = modularPlan.signature
val identifier = dataMapSchema.getRelationIdentifier
val output = new FindDataSourceTable(sparkSession).apply(sparkSession.sessionState.catalog
@@ -138,13 +140,78 @@ private[mv] class SummaryDatasetCatalog(sparkSession: SparkSession)
override def listAllSchema(): Array[SummaryDataset] = summaryDatasets.toArray
+ /**
+ * Registers the data produced by the logical representation of the given [[DataFrame]]. Unlike
+ * `RDD.cache()`, the default storage level is set to be `MEMORY_AND_DISK` because recomputing
+ * the in-memory columnar representation of the underlying table is expensive.
+ */
+ private[mv] def registerSummaryDataset(
+ query: DataFrame,
+ tableName: Option[String] = None): Unit = {
+ writeLock {
+ val planToRegister = query.queryExecution.analyzed
+ if (lookupSummaryDataset(planToRegister).nonEmpty) {
+ sys.error(s"Asked to register already registered.")
+ } else {
+ val modularPlan =
+ mvSession.sessionState.modularizer.modularize(
+ mvSession.sessionState.optimizer.execute(planToRegister)).next()
+ // .harmonized
+ val signature = modularPlan.signature
+ summaryDatasets +=
+ SummaryDataset(signature, planToRegister, null, null)
+ }
+ }
+ }
+
+ /** Removes the given [[DataFrame]] from the catalog */
+ private[mv] def unregisterSummaryDataset(query: DataFrame): Unit = {
+ writeLock {
+ val planToRegister = query.queryExecution.analyzed
+ val dataIndex = summaryDatasets.indexWhere(sd => planToRegister.sameResult(sd.plan))
+ require(dataIndex >= 0, s"Table $query is not registered.")
+ summaryDatasets.remove(dataIndex)
+ }
+ }
+
+ /** Tries to remove the data set for the given [[DataFrame]] from the catalog if it's
+ * registered */
+ private[mv] def tryUnregisterSummaryDataset(
+ query: DataFrame,
+ blocking: Boolean = true): Boolean = {
+ writeLock {
+ val planToRegister = query.queryExecution.analyzed
+ val dataIndex = summaryDatasets.indexWhere(sd => planToRegister.sameResult(sd.plan))
+ val found = dataIndex >= 0
+ if (found) {
+ summaryDatasets.remove(dataIndex)
+ }
+ found
+ }
+ }
+
+ /** Optionally returns registered data set for the given [[DataFrame]] */
+ private[mv] def lookupSummaryDataset(query: DataFrame): Option[SummaryDataset] = {
+ readLock {
+ lookupSummaryDataset(query.queryExecution.analyzed)
+ }
+ }
+
+ /** Returns feasible registered summary data sets for processing the given ModularPlan. */
+ private[mv] def lookupSummaryDataset(plan: LogicalPlan): Option[SummaryDataset] = {
+ readLock {
+ summaryDatasets.find(sd => plan.sameResult(sd.plan))
+ }
+ }
+
+
/** Returns feasible registered summary data sets for processing the given ModularPlan. */
private[mv] def lookupFeasibleSummaryDatasets(plan: ModularPlan): Seq[SummaryDataset] = {
readLock {
val sig = plan.signature
val statusDetails = DataMapStatusManager.getEnabledDataMapStatusDetails
// Only select the enabled datamaps for the query.
- val enabledDataSets = summaryDatasets.filter{p =>
+ val enabledDataSets = summaryDatasets.filter { p =>
statusDetails.exists(_.getDataMapName.equalsIgnoreCase(p.dataMapSchema.getDataMapName))
}
val feasible = enabledDataSets.filter { x =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/Utils.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/Utils.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/Utils.scala
index 074d369..d8af8ab 100644
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/Utils.scala
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/Utils.scala
@@ -17,7 +17,7 @@
package org.apache.carbondata.mv.rewrite
-import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, Expression, PredicateHelper}
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, Cast, Divide, Expression, Multiply, PredicateHelper}
import org.apache.spark.sql.catalyst.expressions.aggregate._
import org.apache.carbondata.mv.plans.modular
@@ -26,7 +26,7 @@ import org.apache.carbondata.mv.plans.modular.ModularPlan
/**
* Utility functions used by mqo matcher to convert our plan to new aggregation code path
*/
-private[rewrite] object Utils extends PredicateHelper {
+object Utils extends PredicateHelper {
// use for match qb_2a, qb_2q and sel_3a, sel_3q
private def doMatch(
@@ -159,7 +159,7 @@ private[rewrite] object Utils extends PredicateHelper {
alias_m(attr).child.asInstanceOf[AggregateExpression]
.aggregateFunction.isInstanceOf[Min] => {
val min_a = alias_m(attr).child.asInstanceOf[AggregateExpression]
- val expr_a = min_a.aggregateFunction.asInstanceOf[Max].child
+ val expr_a = min_a.aggregateFunction.asInstanceOf[Min].child
if (min_a.isDistinct != min_q.isDistinct) {
false
} else {
@@ -174,6 +174,108 @@ private[rewrite] object Utils extends PredicateHelper {
min_q.resultId)
}.getOrElse { matchable = false; min_q }
+
+ case avg_q@AggregateExpression(Average(expr_q), _, false, _) =>
+ val cnt_q = operator_a.outputList.find {
+ case alias: Alias if alias_m.contains(alias.toAttribute) &&
+ alias_m(alias.toAttribute).child.isInstanceOf[AggregateExpression] &&
+ alias_m(alias.toAttribute).child.asInstanceOf[AggregateExpression]
+ .aggregateFunction.isInstanceOf[Count] => { // case for groupby
+ val cnt_a = alias_m(alias.toAttribute).child.asInstanceOf[AggregateExpression]
+ val exprs_a = cnt_a.aggregateFunction.asInstanceOf[Count].children
+ if (!cnt_a.isDistinct && exprs_a.sameElements(Set(expr_q))) {
+ true
+ } else {
+ false
+ }
+ }
+ case attr: Attribute if alias_m.contains(attr) &&
+ alias_m(attr).child.isInstanceOf[AggregateExpression] &&
+ alias_m(attr).child.asInstanceOf[AggregateExpression]
+ .aggregateFunction.isInstanceOf[Count] => {
+ val cnt_a = alias_m(attr).child.asInstanceOf[AggregateExpression]
+ val exprs_a = cnt_a.aggregateFunction.asInstanceOf[Count].children
+ if (!cnt_a.isDistinct && exprs_a.sameElements(Set(expr_q))) {
+ true
+ } else {
+ false
+ }
+ }
+ case _ => false
+ }.map { cnt => Sum(cnt.toAttribute) }
+ .getOrElse { matchable = false; NoOp }
+
+ val derivative = if (matchable) {
+ operator_a.outputList.find {
+ case alias: Alias if alias_m.contains(alias.toAttribute) &&
+ alias_m(alias.toAttribute).child
+ .isInstanceOf[AggregateExpression] &&
+ alias_m(alias.toAttribute).child.asInstanceOf[AggregateExpression]
+ .aggregateFunction.isInstanceOf[Sum] => {
+ val sum_a = alias_m(alias.toAttribute).child.asInstanceOf[AggregateExpression]
+ val expr_a = sum_a.aggregateFunction.asInstanceOf[Sum].child
+ if (sum_a.isDistinct != avg_q.isDistinct) {
+ false
+ } else {
+ expr_a.semanticEquals(expr_q)
+ }
+ }
+ case attr: Attribute if alias_m.contains(attr) &&
+ alias_m(attr).child.isInstanceOf[AggregateExpression] &&
+ alias_m(attr).child.asInstanceOf[AggregateExpression]
+ .aggregateFunction.isInstanceOf[Sum] => {
+ val sum_a = alias_m(attr).child.asInstanceOf[AggregateExpression]
+ val expr_a = sum_a.aggregateFunction.asInstanceOf[Sum].child
+ if (sum_a.isDistinct != avg_q.isDistinct) {
+ false
+ } else {
+ expr_a.semanticEquals(expr_q)
+ }
+ }
+ case alias: Alias if alias_m.contains(alias.toAttribute) &&
+ alias_m(alias.toAttribute).child
+ .isInstanceOf[AggregateExpression] &&
+ alias_m(alias.toAttribute).child.asInstanceOf[AggregateExpression]
+ .aggregateFunction.isInstanceOf[Average] => {
+ val avg_a = alias_m(alias.toAttribute).child.asInstanceOf[AggregateExpression]
+ val expr_a = avg_a.aggregateFunction.asInstanceOf[Average].child
+ if (avg_a.isDistinct != avg_q.isDistinct) {
+ false
+ } else {
+ expr_a.semanticEquals(expr_q)
+ }
+ }
+ case attr: Attribute if alias_m.contains(attr) &&
+ alias_m(attr).child.isInstanceOf[AggregateExpression] &&
+ alias_m(attr).child.asInstanceOf[AggregateExpression]
+ .aggregateFunction.isInstanceOf[Average] => {
+ val avg_a = alias_m(attr).child.asInstanceOf[AggregateExpression]
+ val expr_a = avg_a.aggregateFunction.asInstanceOf[Average].child
+ if (avg_a.isDistinct != avg_q.isDistinct) {
+ false
+ } else {
+ expr_a.semanticEquals(expr_q)
+ }
+ }
+ case _ => false
+ }.map { sum_or_avg =>
+ val fun = alias_m(sum_or_avg.toAttribute).child.asInstanceOf[AggregateExpression]
+ .aggregateFunction
+ if (fun.isInstanceOf[Sum]) {
+ val accu = Sum(sum_or_avg.toAttribute)
+ Divide(accu, Cast(cnt_q, accu.dataType))
+ } else {
+ val accu = Sum(Multiply(sum_or_avg.toAttribute, Cast(cnt_q, sum_or_avg.dataType)))
+ Divide(accu, Cast(cnt_q, accu.dataType))
+ }
+ }
+ } else {
+ matchable = false
+ None
+ }
+
+ derivative.getOrElse { matchable = false; avg_q }
+
case other: AggregateExpression =>
matchable = false
other
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/session/MVSession.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/session/MVSession.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/session/MVSession.scala
new file mode 100644
index 0000000..bcb4d30
--- /dev/null
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/session/MVSession.scala
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.session
+
+import java.io.Closeable
+import java.math.BigInteger
+
+import scala.collection.mutable
+import scala.util.{Failure, Success, Try}
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+import org.apache.carbondata.mv.rewrite.{QueryRewrite, SummaryDatasetCatalog}
+import org.apache.carbondata.mv.session.internal.SessionState
+
+/**
+ * The entry point for working with multi-query optimization in Sparky. Allow the
+ * creation of CSEs (covering subexpression) as well as query rewrite before
+ * submitting to SparkSQL
+ */
+class MVSession(
+ @transient val sparkSession: SparkSession,
+ @transient val catalog: SummaryDatasetCatalog)
+ extends Serializable with Closeable {
+
+ self =>
+
+ /* ----------------------- *
+ | Session-related state |
+ * ----------------------- */
+
+ /**
+ * State isolated across sessions, including SQL configurations, temporary tables, registered
+ * functions, and everything else that accepts a [[org.apache.spark.sql.internal.SQLConf]].
+ */
+ @transient
+ private[mv] lazy val sessionState: SessionState = new SessionState(self)
+
+ @transient
+ lazy val tableFrequencyMap = new mutable.HashMap[String, Int]
+
+ @transient
+ lazy val consumersMap = new mutable.HashMap[BigInteger, mutable.Set[LogicalPlan]] with mutable
+ .MultiMap[BigInteger, LogicalPlan]
+
+ def rewrite(analyzed: LogicalPlan): QueryRewrite = {
+ sessionState.rewritePlan(analyzed)
+ }
+
+ def rewriteToSQL(analyzed: LogicalPlan): String = {
+ val queryRewrite = rewrite(analyzed)
+ Try(queryRewrite.withSummaryData) match {
+ case Success(rewrittenPlan) =>
+ if (rewrittenPlan.fastEquals(queryRewrite.modularPlan)) {
+ ""
+ } else {
+ Try(rewrittenPlan.asCompactSQL) match {
+ case Success(s) => s
+ case Failure(e) => ""
+ }
+ }
+ case Failure(e) => ""
+ }
+ }
+
+ override def close(): Unit = sparkSession.close()
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/session/internal/SessionState.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/session/internal/SessionState.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/session/internal/SessionState.scala
new file mode 100644
index 0000000..993ade9
--- /dev/null
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/session/internal/SessionState.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.mv.session.internal
+
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+import org.apache.carbondata.mv.plans.modular.SimpleModularizer
+import org.apache.carbondata.mv.plans.util.BirdcageOptimizer
+import org.apache.carbondata.mv.rewrite.{DefaultMatchMaker, Navigator, QueryRewrite}
+import org.apache.carbondata.mv.session.MVSession
+
+/**
+ * A class that holds all session-specific state in a given [[MVSession]].
+ */
+private[mv] class SessionState(mvSession: MVSession) {
+
+ // Note: These are all lazy vals because they depend on each other (e.g. conf) and we
+ // want subclasses to override some of the fields. Otherwise, we would get a lot of NPEs.
+
+ /**
+ * Internal catalog for managing table and database states.
+ */
+ lazy val catalog = mvSession.catalog
+
+ /**
+ * Modular query plan modularizer
+ */
+ lazy val modularizer = SimpleModularizer
+
+ /**
+ * Logical query plan optimizer.
+ */
+ lazy val optimizer = BirdcageOptimizer
+
+ lazy val matcher = DefaultMatchMaker
+
+ lazy val navigator: Navigator = new Navigator(catalog, mvSession)
+
+
+ def rewritePlan(plan: LogicalPlan): QueryRewrite = new QueryRewrite(mvSession, plan)
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVCreateTestCase.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVCreateTestCase.scala b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVCreateTestCase.scala
index 4b636db..0aa7b30 100644
--- a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVCreateTestCase.scala
+++ b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVCreateTestCase.scala
@@ -336,7 +336,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
test("test create datamap with simple join") {
sql("drop datamap if exists datamap21")
- sql("create datamap datamap21 using 'mv' as select t1.empname as c1, t2.designation, t2.empname as c2 from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname")
+ sql("create datamap datamap21 using 'mv' as select t1.empname as c1, t2.designation, t2.empname as c2 from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname)")
sql(s"rebuild datamap datamap21")
val frame = sql(
"select t1.empname as c1, t2.designation from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname")
@@ -348,7 +348,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
test("test create datamap with simple join and filter on query") {
sql("drop datamap if exists datamap22")
- sql("create datamap datamap22 using 'mv' as select t1.empname, t2.designation, t2.empname from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname")
+ sql("create datamap datamap22 using 'mv' as select t1.empname, t2.designation from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname)")
sql(s"rebuild datamap datamap22")
val frame = sql(
"select t1.empname, t2.designation from fact_table1 t1,fact_table2 t2 where t1.empname = " +
@@ -363,7 +363,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
test("test create datamap with simple join and filter on query and datamap") {
sql("drop datamap if exists datamap23")
- sql("create datamap datamap23 using 'mv' as select t1.empname, t2.designation from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname and t1.empname='shivani'")
+ sql("create datamap datamap23 using 'mv' as select t1.empname, t2.designation from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) where t1.empname='shivani'")
sql(s"rebuild datamap datamap23")
val frame = sql(
"select t1.empname, t2.designation from fact_table1 t1,fact_table2 t2 where t1.empname = " +
@@ -377,7 +377,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
test("test create datamap with simple join and filter on datamap and no filter on query") {
sql("drop datamap if exists datamap24")
- sql("create datamap datamap24 using 'mv' as select t1.empname, t2.designation from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname and t1.empname='shivani'")
+ sql("create datamap datamap24 using 'mv' as select t1.empname, t2.designation from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) where t1.empname='shivani'")
sql(s"rebuild datamap datamap24")
val frame = sql(
"select t1.empname, t2.designation from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname")
@@ -389,7 +389,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
test("test create datamap with multiple join") {
sql("drop datamap if exists datamap25")
- sql("create datamap datamap25 using 'mv' as select t1.empname as c1, t2.designation from fact_table1 t1,fact_table2 t2,fact_table3 t3 where t1.empname = t2.empname and t1.empname=t3.empname")
+ sql("create datamap datamap25 using 'mv' as select t1.empname as c1, t2.designation from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) inner join fact_table3 t3 on (t1.empname=t3.empname)")
sql(s"rebuild datamap datamap25")
val frame = sql(
"select t1.empname as c1, t2.designation from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname")
@@ -400,20 +400,20 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
}
ignore("test create datamap with simple join on datamap and multi join on query") {
- sql("create datamap datamap26 using 'mv' as select t1.empname, t2.designation, t2.empname from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname")
+ sql("create datamap datamap26 using 'mv' as select t1.empname, t2.designation from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname)")
sql(s"rebuild datamap datamap26")
val frame = sql(
- "select t1.empname, t2.designation, t2.empname from fact_table1 t1,fact_table2 t2,fact_table3 " +
+ "select t1.empname, t2.designation from fact_table1 t1,fact_table2 t2,fact_table3 " +
"t3 where t1.empname = t2.empname and t1.empname=t3.empname")
val analyzed = frame.queryExecution.analyzed
assert(verifyMVDataMap(analyzed, "datamap26"))
- checkAnswer(frame, sql("select t1.empname, t2.designation, t2.empname from fact_table4 t1,fact_table5 t2,fact_table6 " +
+ checkAnswer(frame, sql("select t1.empname, t2.designation from fact_table4 t1,fact_table5 t2,fact_table6 " +
"t3 where t1.empname = t2.empname and t1.empname=t3.empname"))
sql(s"drop datamap datamap26")
}
test("test create datamap with join with group by") {
- sql("create datamap datamap27 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname group by t1.empname, t2.designation")
+ sql("create datamap datamap27 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) group by t1.empname, t2.designation")
sql(s"rebuild datamap datamap27")
val frame = sql(
"select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 " +
@@ -427,7 +427,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
test("test create datamap with join with group by and sub projection") {
sql("drop datamap if exists datamap28")
- sql("create datamap datamap28 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname group by t1.empname, t2.designation")
+ sql("create datamap datamap28 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) group by t1.empname, t2.designation")
sql(s"rebuild datamap datamap28")
val frame = sql(
"select t2.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 where " +
@@ -441,7 +441,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
test("test create datamap with join with group by and sub projection with filter") {
sql("drop datamap if exists datamap29")
- sql("create datamap datamap29 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname group by t1.empname, t2.designation")
+ sql("create datamap datamap29 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) group by t1.empname, t2.designation")
sql(s"rebuild datamap datamap29")
val frame = sql(
"select t2.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 where " +
@@ -453,9 +453,9 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"drop datamap datamap29")
}
- test("test create datamap with join with group by with filter") {
+ ignore("test create datamap with join with group by with filter") {
sql("drop datamap if exists datamap30")
- sql("create datamap datamap30 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 where t1.empname = t2.empname group by t1.empname, t2.designation")
+ sql("create datamap datamap30 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 inner join fact_table2 t2 on (t1.empname = t2.empname) group by t1.empname, t2.designation")
sql(s"rebuild datamap datamap30")
val frame = sql(
"select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1,fact_table2 t2 " +
@@ -467,14 +467,14 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"drop datamap datamap30")
}
- test("test create datamap with expression on projection") {
+ ignore("test create datamap with expression on projection") {
sql(s"drop datamap if exists datamap31")
sql("create datamap datamap31 using 'mv' as select empname, designation, utilization, projectcode from fact_table1 ")
sql(s"rebuild datamap datamap31")
val frame = sql(
"select empname, designation, utilization+projectcode from fact_table1")
val analyzed = frame.queryExecution.analyzed
- assert(verifyMVDataMap(analyzed, "datamap31"))
+ assert(!verifyMVDataMap(analyzed, "datamap31"))
checkAnswer(frame, sql("select empname, designation, utilization+projectcode from fact_table2"))
sql(s"drop datamap datamap31")
}
@@ -501,7 +501,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"drop datamap datamap33")
}
- test("test create datamap with left join with group by") {
+ ignore("test create datamap with left join with group by") {
sql("drop datamap if exists datamap34")
sql("create datamap datamap34 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 on t1.empname = t2.empname group by t1.empname, t2.designation")
sql(s"rebuild datamap datamap34")
@@ -515,7 +515,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"drop datamap datamap34")
}
- test("test create datamap with simple and group by query with filter on datamap but not on projection") {
+ ignore("test create datamap with simple and group by query with filter on datamap but not on projection") {
sql("create datamap datamap35 using 'mv' as select designation, sum(utilization) from fact_table1 where empname='shivani' group by designation")
sql(s"rebuild datamap datamap35")
val frame = sql(
@@ -526,7 +526,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"drop datamap datamap35")
}
- test("test create datamap with simple and sub group by query with filter on datamap but not on projection") {
+ ignore("test create datamap with simple and sub group by query with filter on datamap but not on projection") {
sql("create datamap datamap36 using 'mv' as select designation, sum(utilization) from fact_table1 where empname='shivani' group by designation")
sql(s"rebuild datamap datamap36")
val frame = sql(
@@ -565,7 +565,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"drop datamap datamap38")
}
- test("test create datamap with agg push join with group by with filter") {
+ ignore("test create datamap with agg push join with group by with filter") {
sql("drop datamap if exists datamap39")
sql("create datamap datamap39 using 'mv' as select empname, designation, sum(utilization) from fact_table1 group by empname, designation ")
sql(s"rebuild datamap datamap39")
@@ -593,7 +593,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"drop datamap datamap40")
}
- test("test create datamap with left join with group by with filter") {
+ ignore("test create datamap with left join with group by with filter") {
sql("drop datamap if exists datamap41")
sql("create datamap datamap41 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 on t1.empname = t2.empname group by t1.empname, t2.designation")
sql(s"rebuild datamap datamap41")
@@ -607,7 +607,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"drop datamap datamap41")
}
- test("test create datamap with left join with sub group by") {
+ ignore("test create datamap with left join with sub group by") {
sql("drop datamap if exists datamap42")
sql("create datamap datamap42 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 on t1.empname = t2.empname group by t1.empname, t2.designation")
sql(s"rebuild datamap datamap42")
@@ -621,7 +621,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"drop datamap datamap42")
}
- test("test create datamap with left join with sub group by with filter") {
+ ignore("test create datamap with left join with sub group by with filter") {
sql("drop datamap if exists datamap43")
sql("create datamap datamap43 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 on t1.empname = t2.empname group by t1.empname, t2.designation")
sql(s"rebuild datamap datamap43")
@@ -635,7 +635,7 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"drop datamap datamap43")
}
- test("test create datamap with left join with sub group by with filter on mv") {
+ ignore("test create datamap with left join with sub group by with filter on mv") {
sql("drop datamap if exists datamap44")
sql("create datamap datamap44 using 'mv' as select t1.empname, t2.designation, sum(t1.utilization) from fact_table1 t1 left join fact_table2 t2 on t1.empname = t2.empname where t1.empname='shivani' group by t1.empname, t2.designation")
sql(s"rebuild datamap datamap44")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVTPCDSTestCase.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVTPCDSTestCase.scala b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVTPCDSTestCase.scala
index d7a19b8..b2d03e1 100644
--- a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVTPCDSTestCase.scala
+++ b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVTPCDSTestCase.scala
@@ -68,7 +68,7 @@ class MVTPCDSTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"drop datamap datamap_tpcds3")
}
- test("test create datamap with tpcds_1_4_testCases case_4") {
+ ignore("test create datamap with tpcds_1_4_testCases case_4") {
sql(s"drop datamap if exists datamap_tpcds4")
sql(s"create datamap datamap_tpcds4 using 'mv' as ${tpcds_1_4_testCases(3)._2}")
sql(s"rebuild datamap datamap_tpcds4")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/SelectSelectExactChildrenSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/SelectSelectExactChildrenSuite.scala b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/SelectSelectExactChildrenSuite.scala
index 0ee2475..f84d4c6 100644
--- a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/SelectSelectExactChildrenSuite.scala
+++ b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/SelectSelectExactChildrenSuite.scala
@@ -19,9 +19,10 @@ package org.apache.carbondata.mv.rewrite
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.test.util.PlanTest
-class SelectSelectExactChildrenSuite extends PlanTest {
+import org.apache.carbondata.mv.testutil.ModularPlanTest
+
+class SelectSelectExactChildrenSuite extends ModularPlanTest {
object Match extends DefaultMatchMaker {
val patterns = SelectSelectNoChildDelta :: Nil
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/TestSQLSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/TestSQLSuite.scala b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/TestSQLSuite.scala
new file mode 100644
index 0000000..25f07e4
--- /dev/null
+++ b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/TestSQLSuite.scala
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.rewrite
+
+import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.hive.CarbonSessionCatalog
+import org.scalatest.BeforeAndAfter
+
+import org.apache.carbondata.mv.testutil.ModularPlanTest
+
+class TestSQLSuite extends ModularPlanTest with BeforeAndAfter {
+ import org.apache.carbondata.mv.rewrite.matching.TestSQLBatch._
+
+ val spark = sqlContext
+ val testHive = sqlContext.sparkSession
+ val hiveClient = spark.sparkSession.sessionState.catalog.asInstanceOf[CarbonSessionCatalog].getClient()
+
+ ignore("protypical mqo rewrite test") {
+
+ hiveClient.runSqlHive(
+ s"""
+ |CREATE TABLE if not exists Fact (
+ | `tid` int,
+ | `fpgid` int,
+ | `flid` int,
+ | `date` timestamp,
+ | `faid` int,
+ | `price` double,
+ | `qty` int,
+ | `disc` string
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
+ """.stripMargin.trim
+ )
+
+ hiveClient.runSqlHive(
+ s"""
+ |CREATE TABLE if not exists Dim (
+ | `lid` int,
+ | `city` string,
+ | `state` string,
+ | `country` string
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
+ """.stripMargin.trim
+ )
+
+ hiveClient.runSqlHive(
+ s"""
+ |CREATE TABLE if not exists Item (
+ | `i_item_id` int,
+ | `i_item_sk` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
+ """.stripMargin.trim
+ )
+
+ val dest = "case_11"
+
+ sampleTestCases.foreach { testcase =>
+ if (testcase._1 == dest) {
+ val mvSession = new SummaryDatasetCatalog(testHive)
+ val summary = testHive.sql(testcase._2)
+ mvSession.registerSummaryDataset(summary)
+ val rewrittenSQL =
+ mvSession.mvSession.rewrite(mvSession.mvSession.sparkSession.sql(
+ testcase._3).queryExecution.analyzed).toCompactSQL.trim
+
+ if (!rewrittenSQL.trim.equals(testcase._4)) {
+ fail(
+ s"""
+ |=== FAIL: SQLs do not match ===
+ |${sideBySide(rewrittenSQL, testcase._4).mkString("\n")}
+ """.stripMargin)
+ }
+ }
+
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/Tpcds_1_4_Suite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/Tpcds_1_4_Suite.scala b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/Tpcds_1_4_Suite.scala
new file mode 100644
index 0000000..76e0455
--- /dev/null
+++ b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/Tpcds_1_4_Suite.scala
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.rewrite
+
+import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.hive.CarbonSessionCatalog
+import org.scalatest.BeforeAndAfter
+
+import org.apache.carbondata.mv.testutil.ModularPlanTest
+//import org.apache.spark.sql.catalyst.SQLBuilder
+import java.io.{File, PrintWriter}
+
+class Tpcds_1_4_Suite extends ModularPlanTest with BeforeAndAfter {
+ import org.apache.carbondata.mv.rewrite.matching.TestTPCDS_1_4_Batch._
+ import org.apache.carbondata.mv.testutil.Tpcds_1_4_Tables._
+
+ val spark = sqlContext
+ val testHive = sqlContext.sparkSession
+ val hiveClient = spark.sparkSession.sessionState.catalog.asInstanceOf[CarbonSessionCatalog].getClient()
+
+ ignore("test using tpc-ds queries") {
+
+ tpcds1_4Tables.foreach { create_table =>
+ hiveClient.runSqlHive(create_table)
+ }
+
+ val writer = new PrintWriter(new File("batch.txt"))
+// val dest = "case_30"
+// val dest = "case_32"
+// val dest = "case_33"
+// case_15 and case_16 need revisit
+
+ val dest = "case_29" /** to run single case, uncomment out this **/
+
+ tpcds_1_4_testCases.foreach { testcase =>
+// if (testcase._1 == dest) { /** to run single case, uncomment out this **/
+ val mvSession = new SummaryDatasetCatalog(testHive)
+ val summaryDF = testHive.sql(testcase._2)
+ mvSession.registerSummaryDataset(summaryDF)
+
+ writer.print(s"\n\n==== ${testcase._1} ====\n\n==== mv ====\n\n${testcase._2}\n\n==== original query ====\n\n${testcase._3}\n")
+
+ val rewriteSQL = mvSession.mvSession.rewriteToSQL(mvSession.mvSession.sparkSession.sql(testcase._3).queryExecution.analyzed)
+ LOGGER.info(s"\n\n\n\n===== Rewritten query for ${testcase._1} =====\n\n${rewriteSQL}\n")
+
+ if (!rewriteSQL.trim.equals(testcase._4)) {
+ LOGGER.error(s"===== Rewrite not matched for ${testcase._1}\n")
+ LOGGER.error(s"\n\n===== Rewrite failed for ${testcase._1}, Expected: =====\n\n${testcase._4}\n")
+ LOGGER.error(
+ s"""
+ |=== FAIL: SQLs do not match ===
+ |${sideBySide(rewriteSQL, testcase._4).mkString("\n")}
+ """.stripMargin)
+ writer.print(s"\n\n==== result ====\n\nfailed\n")
+ writer.print(s"\n\n==== rewritten query ====\n\n${rewriteSQL}\n")
+ }
+ else {
+ LOGGER.info(s"===== Rewrite successful for ${testcase._1}, as expected\n")
+ writer.print(s"\n\n==== result ====\n\nsuccessful\n")
+ writer.print(s"\n\n==== rewritten query ====\n\n${rewriteSQL}\n")
+ }
+
+// } /**to run single case, uncomment out this **/
+
+ }
+
+ writer.close()
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestSQLBatch.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestSQLBatch.scala b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestSQLBatch.scala
index 02bbff3..96f1816 100644
--- a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestSQLBatch.scala
+++ b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestSQLBatch.scala
@@ -15,7 +15,6 @@
* limitations under the License.
*/
-
package org.apache.carbondata.mv.rewrite.matching
object TestSQLBatch {
@@ -210,5 +209,27 @@ object TestSQLBatch {
| fact
|WHERE
| ((fact.`faid` > 0) OR (fact.`flid` > 0))
+ """.stripMargin.trim),
+ ("case_11",
+ s"""
+ |SELECT faid, count(flid)
+ |FROM Fact
+ |GROUP BY faid
+ """.stripMargin.trim,
+ s"""
+ |SELECT faid, count(flid)
+ |FROM Fact
+ |WHERE faid = 3
+ |GROUP BY faid
+ """.stripMargin.trim,
+ s"""
+ |SELECT gen_subsumer_0.`faid`, gen_subsumer_0.`count(flid)` AS `count(flid)`
+ |FROM
+ | (SELECT fact.`faid`, count(fact.`flid`) AS `count(flid)`
+ | FROM
+ | fact
+ | GROUP BY fact.`faid`) gen_subsumer_0
+ |WHERE
+ | (gen_subsumer_0.`faid` = 3)
""".stripMargin.trim))
}
\ No newline at end of file
[28/50] [abbrv] carbondata git commit: [CARBONDATA-2428] Support flat
folder for managed carbon table
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 9d0c933..b76722b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -279,7 +279,7 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
this.carbonDataFileName = CarbonTablePath
.getCarbonDataFileName(fileCount, model.getCarbonDataFileAttributes().getTaskId(),
model.getBucketId(), model.getTaskExtension(),
- "" + model.getCarbonDataFileAttributes().getFactTimeStamp());
+ "" + model.getCarbonDataFileAttributes().getFactTimeStamp(), model.getSegmentId());
this.carbonDataFileHdfsPath = model.getCarbonDataDirectoryPath() + File.separator
+ carbonDataFileName;
try {
@@ -368,7 +368,7 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
String rawFileName = model.getCarbonDataDirectoryPath() + File.separator + CarbonTablePath
.getCarbonIndexFileName(model.getCarbonDataFileAttributes().getTaskId(),
model.getBucketId(), model.getTaskExtension(),
- "" + model.getCarbonDataFileAttributes().getFactTimeStamp());
+ "" + model.getCarbonDataFileAttributes().getFactTimeStamp(), model.getSegmentId());
indexFileName = FileFactory.getUpdatedFilePath(rawFileName, FileFactory.FileType.HDFS);
} else {
// randomly choose a temp location for index file
@@ -378,7 +378,7 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
indexFileName = chosenTempLocation + File.separator + CarbonTablePath
.getCarbonIndexFileName(model.getCarbonDataFileAttributes().getTaskId(),
model.getBucketId(), model.getTaskExtension(),
- "" + model.getCarbonDataFileAttributes().getFactTimeStamp());
+ "" + model.getCarbonDataFileAttributes().getFactTimeStamp(), model.getSegmentId());
}
CarbonIndexFileWriter writer = new CarbonIndexFileWriter();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 0ea7223..da77cf6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -601,7 +601,9 @@ public final class CarbonLoaderUtil {
long sizePerNode = 0;
long totalFileSize = 0;
if (BlockAssignmentStrategy.BLOCK_NUM_FIRST == blockAssignmentStrategy) {
- sizePerNode = blockInfos.size() / noofNodes;
+ if (blockInfos.size() > 0) {
+ sizePerNode = blockInfos.size() / noofNodes;
+ }
sizePerNode = sizePerNode <= 0 ? 1 : sizePerNode;
} else if (BlockAssignmentStrategy.BLOCK_SIZE_FIRST == blockAssignmentStrategy
|| BlockAssignmentStrategy.NODE_MIN_SIZE_FIRST == blockAssignmentStrategy) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
----------------------------------------------------------------------
diff --git a/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java b/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
index f6406c7..4bfadce 100644
--- a/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
+++ b/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
@@ -37,15 +37,18 @@ import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
-import org.apache.carbondata.core.readcommitter.LatestFilesReadCommittedScope;
+import org.apache.carbondata.core.readcommitter.TableStatusReadCommittedScope;
import org.apache.carbondata.core.scan.executor.impl.SearchModeDetailQueryExecutor;
import org.apache.carbondata.core.scan.executor.impl.SearchModeVectorDetailQueryExecutor;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.scan.model.QueryModelBuilder;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.util.CarbonTaskInfo;
import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.hadoop.CarbonInputSplit;
import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
import org.apache.carbondata.hadoop.CarbonRecordReader;
@@ -164,12 +167,15 @@ public class SearchRequestHandler {
Objects.requireNonNull(datamap);
List<Segment> segments = new LinkedList<>();
HashMap<String, Integer> uniqueSegments = new HashMap<>();
+ LoadMetadataDetails[] loadMetadataDetails =
+ SegmentStatusManager.readLoadMetadata(
+ CarbonTablePath.getMetadataPath(table.getTablePath()));
for (CarbonInputSplit split : mbSplit.getAllSplits()) {
- String segmentId = split.getSegmentId();
+ String segmentId = Segment.getSegment(split.getSegmentId(), loadMetadataDetails).toString();
if (uniqueSegments.get(segmentId) == null) {
- segments.add(Segment.toSegment(
- segmentId,
- new LatestFilesReadCommittedScope(table.getTablePath(), segmentId)));
+ segments.add(Segment.toSegment(segmentId,
+ new TableStatusReadCommittedScope(table.getAbsoluteTableIdentifier(),
+ loadMetadataDetails)));
uniqueSegments.put(segmentId, 1);
} else {
uniqueSegments.put(segmentId, uniqueSegments.get(segmentId) + 1);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/streaming/src/main/java/org/apache/carbondata/streaming/CarbonStreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/carbondata/streaming/CarbonStreamRecordWriter.java b/streaming/src/main/java/org/apache/carbondata/streaming/CarbonStreamRecordWriter.java
index 4653445..bd622f0 100644
--- a/streaming/src/main/java/org/apache/carbondata/streaming/CarbonStreamRecordWriter.java
+++ b/streaming/src/main/java/org/apache/carbondata/streaming/CarbonStreamRecordWriter.java
@@ -128,7 +128,7 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
segmentDir = CarbonTablePath.getSegmentPath(
carbonTable.getAbsoluteTableIdentifier().getTablePath(), segmentId);
- fileName = CarbonTablePath.getCarbonDataFileName(0, taskNo, 0, 0, "0");
+ fileName = CarbonTablePath.getCarbonDataFileName(0, taskNo, 0, 0, "0", segmentId);
}
private void initializeAtFirstRow() throws IOException, InterruptedException {
[46/50] [abbrv] carbondata git commit: [CARBONDATA-2504][STREAM]
Support StreamSQL for streaming job
Posted by ja...@apache.org.
[CARBONDATA-2504][STREAM] Support StreamSQL for streaming job
Currently, user need to write Spark Streaming APP to use carbon streaming ingest feature, which is not so easy for some users. By providing StreamSQL, user can manage the streaming job more easily.
This closes #2328
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2ea3b2dc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2ea3b2dc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2ea3b2dc
Branch: refs/heads/carbonstore
Commit: 2ea3b2dc5841c604c5fb44a6e7f18c7d2db8c543
Parents: 6eb360e
Author: Jacky Li <ja...@qq.com>
Authored: Mon May 21 21:49:33 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Jun 21 09:44:22 2018 +0530
----------------------------------------------------------------------
.../exceptions/NoSuchStreamException.java | 35 ++
.../core/metadata/schema/table/CarbonTable.java | 15 +-
.../spark/util/SparkDataTypeConverterImpl.java | 45 +++
.../apache/carbondata/spark/CarbonOption.scala | 9 +-
.../carbondata/spark/StreamingOption.scala | 67 ++++
.../carbondata/spark/util/CarbonScalaUtil.scala | 5 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 3 +
.../spark/util/CarbonMetastoreTypes.scala | 104 ++++++
.../apache/spark/util/SparkTypeConverter.scala | 134 +++++++
.../carbondata/stream/StreamJobManager.scala | 198 ++++++++++
.../org/apache/spark/sql/CarbonSource.scala | 2 +-
.../spark/sql/CarbonSparkStreamingFactory.scala | 2 +-
.../datamap/CarbonCreateDataMapCommand.scala | 2 +-
.../stream/CarbonCreateStreamCommand.scala | 130 +++++++
.../stream/CarbonDropStreamCommand.scala | 36 ++
.../stream/CarbonShowStreamsCommand.scala | 76 ++++
.../command/table/CarbonDropTableCommand.scala | 2 +-
.../strategy/CarbonLateDecodeStrategy.scala | 2 +-
.../sql/execution/strategy/DDLStrategy.scala | 2 +-
.../strategy/StreamingTableStrategy.scala | 2 +-
.../sql/hive/CarbonPreAggregateRules.scala | 6 +-
.../apache/spark/sql/hive/CarbonRelation.scala | 89 +----
.../sql/parser/CarbonSpark2SqlParser.scala | 44 ++-
.../spark/sql/parser/CarbonSparkSqlParser.scala | 4 +-
.../apache/spark/util/SparkTypeConverter.scala | 135 -------
.../TestStreamingTableOperation.scala | 359 ++++++++++++++++++-
.../carbondata/store/LocalCarbonStore.java | 2 +-
.../streaming/parser/RowStreamParserImp.scala | 1 -
28 files changed, 1254 insertions(+), 257 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/common/src/main/java/org/apache/carbondata/common/exceptions/NoSuchStreamException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/exceptions/NoSuchStreamException.java b/common/src/main/java/org/apache/carbondata/common/exceptions/NoSuchStreamException.java
new file mode 100644
index 0000000..77fa7fb
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/exceptions/NoSuchStreamException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.exceptions;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+@InterfaceAudience.User
+@InterfaceStability.Stable
+public class NoSuchStreamException extends Exception {
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 133241232L;
+
+ public NoSuchStreamException(String streamName) {
+ super("stream '" + streamName + "' not found");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index b7bef28..c302b2b 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -874,11 +874,20 @@ public class CarbonTable implements Serializable {
}
/**
- * Return true if this is a streaming table (table with property "streaming"="true")
+ * Return true if this is a streaming table (table with property "streaming"="true" or "sink")
*/
- public boolean isStreamingTable() {
+ public boolean isStreamingSink() {
String streaming = getTableInfo().getFactTable().getTableProperties().get("streaming");
- return streaming != null && streaming.equalsIgnoreCase("true");
+ return streaming != null &&
+ (streaming.equalsIgnoreCase("true") || streaming.equalsIgnoreCase("sink"));
+ }
+
+ /**
+ * Return true if this is a streaming source (table with property "streaming"="source")
+ */
+ public boolean isStreamingSource() {
+ String streaming = getTableInfo().getFactTable().getTableProperties().get("streaming");
+ return streaming != null && streaming.equalsIgnoreCase("source");
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/SparkDataTypeConverterImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/SparkDataTypeConverterImpl.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/SparkDataTypeConverterImpl.java
index f6dc65b..3951642 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/SparkDataTypeConverterImpl.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/SparkDataTypeConverterImpl.java
@@ -19,21 +19,29 @@ package org.apache.carbondata.spark.util;
import java.io.Serializable;
import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.DataTypeConverter;
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
import org.apache.spark.sql.catalyst.util.GenericArrayData;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.DecimalType;
+import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
import org.apache.spark.unsafe.types.UTF8String;
+import org.apache.spark.util.CarbonMetastoreTypes;
+import org.apache.spark.util.SparkTypeConverter;
/**
* Convert java data type to spark data type
@@ -171,4 +179,41 @@ public final class SparkDataTypeConverterImpl implements DataTypeConverter, Seri
}
return fields;
}
+
+ public static StructType convertToSparkSchema(CarbonTable table, ColumnSchema[] carbonColumns) {
+ List<StructField> fields = new ArrayList<>(carbonColumns.length);
+ for (int i = 0; i < carbonColumns.length; i++) {
+ ColumnSchema carbonColumn = carbonColumns[i];
+ DataType dataType = carbonColumn.getDataType();
+ if (org.apache.carbondata.core.metadata.datatype.DataTypes.isDecimal(dataType)) {
+ fields.add(new StructField(carbonColumn.getColumnName(),
+ new DecimalType(carbonColumn.getPrecision(), carbonColumn.getScale()),
+ true, Metadata.empty()));
+ } else if (org.apache.carbondata.core.metadata.datatype.DataTypes.isStructType(dataType)) {
+ fields.add(
+ new StructField(
+ carbonColumn.getColumnName(),
+ CarbonMetastoreTypes.toDataType(
+ String.format("struct<%s>",
+ SparkTypeConverter.getStructChildren(table, carbonColumn.getColumnName()))),
+ true,
+ Metadata.empty()));
+ } else if (org.apache.carbondata.core.metadata.datatype.DataTypes.isArrayType(dataType)) {
+ fields.add(
+ new StructField(
+ carbonColumn.getColumnName(),
+ CarbonMetastoreTypes.toDataType(
+ String.format("array<%s>",
+ SparkTypeConverter.getArrayChildren(
+ table,
+ carbonColumn.getColumnName()))),
+ true,
+ Metadata.empty()));
+ } else {
+ fields.add(new StructField(carbonColumn.getColumnName(),
+ convertCarbonToSparkDataType(carbonColumn.getDataType()), true, Metadata.empty()));
+ }
+ }
+ return new StructType(fields.toArray(new StructField[0]));
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
index e854bbe..a48e63d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
@@ -57,8 +57,13 @@ class CarbonOption(options: Map[String, String]) {
def isBucketingEnabled: Boolean = options.contains("bucketcolumns") &&
options.contains("bucketnumber")
- def isStreaming: Boolean =
- options.getOrElse("streaming", "false").toBoolean
+ def isStreaming: Boolean = {
+ var stream = options.getOrElse("streaming", "false")
+ if (stream.equalsIgnoreCase("sink") || stream.equalsIgnoreCase("source")) {
+ stream = "true"
+ }
+ stream.toBoolean
+ }
def overwriteEnabled: Boolean =
options.getOrElse("overwrite", "false").toBoolean
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/StreamingOption.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/StreamingOption.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/StreamingOption.scala
new file mode 100644
index 0000000..c724474
--- /dev/null
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/StreamingOption.scala
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark
+
+import scala.collection.mutable
+
+import org.apache.spark.sql.streaming.{ProcessingTime, Trigger}
+
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.streaming.parser.CarbonStreamParser
+
+class StreamingOption(val userInputMap: Map[String, String]) {
+ lazy val trigger: Trigger = {
+ val trigger = userInputMap.getOrElse(
+ "trigger", throw new MalformedCarbonCommandException("trigger must be specified"))
+ val interval = userInputMap.getOrElse(
+ "interval", throw new MalformedCarbonCommandException("interval must be specified"))
+ trigger match {
+ case "ProcessingTime" => ProcessingTime(interval)
+ case others => throw new MalformedCarbonCommandException("invalid trigger: " + trigger)
+ }
+ }
+
+ def checkpointLocation(tablePath: String): String =
+ userInputMap.getOrElse(
+ "checkpointLocation",
+ CarbonTablePath.getStreamingCheckpointDir(tablePath))
+
+ lazy val timeStampFormat: String =
+ userInputMap.getOrElse("timestampformat", CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+
+ lazy val dateFormat: String =
+ userInputMap.getOrElse("dateformat", CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
+
+ lazy val rowParser: String =
+ userInputMap.getOrElse(CarbonStreamParser.CARBON_STREAM_PARSER,
+ CarbonStreamParser.CARBON_STREAM_PARSER_ROW_PARSER)
+
+ lazy val remainingOption: Map[String, String] = {
+ // copy the user input map and remove the fix options
+ val mutableMap = mutable.Map[String, String]() ++= userInputMap
+ mutableMap.remove("checkpointLocation")
+ mutableMap.remove("timestampformat")
+ mutableMap.remove("dateformat")
+ mutableMap.remove("trigger")
+ mutableMap.remove("interval")
+ mutableMap.remove(CarbonStreamParser.CARBON_STREAM_PARSER)
+ mutableMap.toMap
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 6227655..3e94a66 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -42,7 +42,7 @@ import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumn
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory
import org.apache.carbondata.core.metadata.ColumnIdentifier
-import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, StructField => CarbonStructField}
+import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, DecimalType => CarbonDecimalType, StructField => CarbonStructField}
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema}
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, ColumnSchema}
@@ -104,7 +104,8 @@ object CarbonScalaUtil {
def convertCarbonToSparkDataType(dataType: CarbonDataType): types.DataType = {
if (CarbonDataTypes.isDecimal(dataType)) {
- DecimalType.SYSTEM_DEFAULT
+ DecimalType(dataType.asInstanceOf[CarbonDecimalType].getPrecision,
+ dataType.asInstanceOf[CarbonDecimalType].getScale)
} else {
dataType match {
case CarbonDataTypes.STRING => StringType
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 0d53a73..350fc36 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -184,6 +184,9 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
protected val SELECT = carbonKeyWord("SELECT")
protected val REBUILD = carbonKeyWord("REBUILD")
protected val DEFERRED = carbonKeyWord("DEFERRED")
+ protected val STREAM = carbonKeyWord("STREAM")
+ protected val STREAMS = carbonKeyWord("STREAMS")
+ protected val STMPROPERTIES = carbonKeyWord("STMPROPERTIES")
protected val doubleQuotedString = "\"([^\"]+)\"".r
protected val singleQuotedString = "'([^']+)'".r
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonMetastoreTypes.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonMetastoreTypes.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonMetastoreTypes.scala
new file mode 100644
index 0000000..6dbd3a3
--- /dev/null
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonMetastoreTypes.scala
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import scala.util.parsing.combinator.RegexParsers
+
+import org.apache.spark.sql.types._
+import org.apache.spark.sql.util.CarbonException
+
+object CarbonMetastoreTypes extends RegexParsers {
+ protected lazy val primitiveType: Parser[DataType] =
+ "string" ^^^ StringType |
+ "varchar" ^^^ StringType |
+ "float" ^^^ FloatType |
+ "int" ^^^ IntegerType |
+ "tinyint" ^^^ ShortType |
+ "short" ^^^ ShortType |
+ "double" ^^^ DoubleType |
+ "long" ^^^ LongType |
+ "binary" ^^^ BinaryType |
+ "boolean" ^^^ BooleanType |
+ fixedDecimalType |
+ "decimal" ^^^ "decimal" ^^^ DecimalType(10, 0) |
+ "varchar\\((\\d+)\\)".r ^^^ StringType |
+ "date" ^^^ DateType |
+ "timestamp" ^^^ TimestampType
+
+ protected lazy val fixedDecimalType: Parser[DataType] =
+ "decimal" ~> "(" ~> "^[1-9]\\d*".r ~ ("," ~> "^[0-9]\\d*".r <~ ")") ^^ {
+ case precision ~ scale =>
+ DecimalType(precision.toInt, scale.toInt)
+ }
+
+ protected lazy val arrayType: Parser[DataType] =
+ "array" ~> "<" ~> dataType <~ ">" ^^ {
+ case tpe => ArrayType(tpe)
+ }
+
+ protected lazy val mapType: Parser[DataType] =
+ "map" ~> "<" ~> dataType ~ "," ~ dataType <~ ">" ^^ {
+ case t1 ~ _ ~ t2 => MapType(t1, t2)
+ }
+
+ protected lazy val structField: Parser[StructField] =
+ "[a-zA-Z0-9_]*".r ~ ":" ~ dataType ^^ {
+ case name ~ _ ~ tpe => StructField(name, tpe, nullable = true)
+ }
+
+ protected lazy val structType: Parser[DataType] =
+ "struct" ~> "<" ~> repsep(structField, ",") <~ ">" ^^ {
+ case fields => StructType(fields)
+ }
+
+ protected lazy val dataType: Parser[DataType] =
+ arrayType |
+ mapType |
+ structType |
+ primitiveType
+
+ def toDataType(metastoreType: String): DataType = {
+ parseAll(dataType, metastoreType) match {
+ case Success(result, _) => result
+ case _: NoSuccess =>
+ CarbonException.analysisException(s"Unsupported dataType: $metastoreType")
+ }
+ }
+
+ def toMetastoreType(dt: DataType): String = {
+ dt match {
+ case ArrayType(elementType, _) => s"array<${ toMetastoreType(elementType) }>"
+ case StructType(fields) =>
+ s"struct<${
+ fields.map(f => s"${ f.name }:${ toMetastoreType(f.dataType) }")
+ .mkString(",")
+ }>"
+ case StringType => "string"
+ case FloatType => "float"
+ case IntegerType => "int"
+ case ShortType => "tinyint"
+ case DoubleType => "double"
+ case LongType => "bigint"
+ case BinaryType => "binary"
+ case BooleanType => "boolean"
+ case DecimalType() => "decimal"
+ case TimestampType => "timestamp"
+ case DateType => "date"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark-common/src/main/scala/org/apache/spark/util/SparkTypeConverter.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/SparkTypeConverter.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/SparkTypeConverter.scala
new file mode 100644
index 0000000..8a9277c
--- /dev/null
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/SparkTypeConverter.scala
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import java.util.Objects
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.types
+import org.apache.spark.sql.types._
+
+import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension, ColumnSchema}
+
+private[spark] object SparkTypeConverter {
+
+ def createSparkSchema(table: CarbonTable, columns: Seq[String]): StructType = {
+ Objects.requireNonNull(table)
+ Objects.requireNonNull(columns)
+ if (columns.isEmpty) {
+ throw new IllegalArgumentException("column list is empty")
+ }
+ val fields = new java.util.ArrayList[StructField](columns.size)
+ val allColumns = table.getTableInfo.getFactTable.getListOfColumns.asScala
+
+ // find the column and add it to fields array
+ columns.foreach { column =>
+ val col = allColumns.find(_.getColumnName.equalsIgnoreCase(column)).getOrElse(
+ throw new IllegalArgumentException(column + " does not exist")
+ )
+ fields.add(StructField(col.getColumnName, convertCarbonToSparkDataType(col, table)))
+ }
+ StructType(fields)
+ }
+
+ /**
+ * Converts from carbon datatype to corresponding spark datatype.
+ */
+ def convertCarbonToSparkDataType(
+ columnSchema: ColumnSchema,
+ table: CarbonTable): types.DataType = {
+ if (CarbonDataTypes.isDecimal(columnSchema.getDataType)) {
+ val scale = columnSchema.getScale
+ val precision = columnSchema.getPrecision
+ if (scale == 0 && precision == 0) {
+ DecimalType(18, 2)
+ } else {
+ DecimalType(precision, scale)
+ }
+ } else if (CarbonDataTypes.isArrayType(columnSchema.getDataType)) {
+ CarbonMetastoreTypes
+ .toDataType(s"array<${ getArrayChildren(table, columnSchema.getColumnName) }>")
+ } else if (CarbonDataTypes.isStructType(columnSchema.getDataType)) {
+ CarbonMetastoreTypes
+ .toDataType(s"struct<${ getStructChildren(table, columnSchema.getColumnName) }>")
+ } else {
+ columnSchema.getDataType match {
+ case CarbonDataTypes.STRING => StringType
+ case CarbonDataTypes.SHORT => ShortType
+ case CarbonDataTypes.INT => IntegerType
+ case CarbonDataTypes.LONG => LongType
+ case CarbonDataTypes.DOUBLE => DoubleType
+ case CarbonDataTypes.BOOLEAN => BooleanType
+ case CarbonDataTypes.TIMESTAMP => TimestampType
+ case CarbonDataTypes.DATE => DateType
+ }
+ }
+ }
+
+ def getArrayChildren(table: CarbonTable, dimName: String): String = {
+ table.getChildren(dimName).asScala.map(childDim => {
+ childDim.getDataType.getName.toLowerCase match {
+ case "array" => s"array<${ getArrayChildren(table, childDim.getColName) }>"
+ case "struct" => s"struct<${ getStructChildren(table, childDim.getColName) }>"
+ case dType => addDecimalScaleAndPrecision(childDim, dType)
+ }
+ }).mkString(",")
+ }
+
+ def getStructChildren(table: CarbonTable, dimName: String): String = {
+ table.getChildren(dimName).asScala.map(childDim => {
+ childDim.getDataType.getName.toLowerCase match {
+ case "array" => s"${
+ childDim.getColName.substring(dimName.length + 1)
+ }:array<${ getArrayChildren(table, childDim.getColName) }>"
+ case "struct" => s"${
+ childDim.getColName.substring(dimName.length + 1)
+ }:struct<${ table.getChildren(childDim.getColName)
+ .asScala.map(f => s"${ recursiveMethod(table, childDim.getColName, f) }").mkString(",")
+ }>"
+ case dType => s"${ childDim.getColName
+ .substring(dimName.length() + 1) }:${ addDecimalScaleAndPrecision(childDim, dType) }"
+ }
+ }).mkString(",")
+ }
+
+ def addDecimalScaleAndPrecision(dimval: CarbonColumn, dataType: String): String = {
+ var dType = dataType
+ if (CarbonDataTypes.isDecimal(dimval.getDataType)) {
+ dType +=
+ "(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema.getScale + ")"
+ }
+ dType
+ }
+
+ private def recursiveMethod(
+ table: CarbonTable, dimName: String, childDim: CarbonDimension) = {
+ childDim.getDataType.getName.toLowerCase match {
+ case "array" => s"${
+ childDim.getColName.substring(dimName.length + 1)
+ }:array<${ getArrayChildren(table, childDim.getColName) }>"
+ case "struct" => s"${
+ childDim.getColName.substring(dimName.length + 1)
+ }:struct<${ getStructChildren(table, childDim.getColName) }>"
+ case dType => s"${ childDim.getColName.substring(dimName.length + 1) }:${ dType }"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/carbondata/stream/StreamJobManager.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/stream/StreamJobManager.scala b/integration/spark2/src/main/scala/org/apache/carbondata/stream/StreamJobManager.scala
new file mode 100644
index 0000000..59e924d
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/stream/StreamJobManager.scala
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.stream
+
+import java.util.concurrent.{ConcurrentHashMap, CountDownLatch, TimeUnit}
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.streaming.StreamingQuery
+import org.apache.spark.sql.types.{StructField, StructType}
+
+import org.apache.carbondata.common.exceptions.NoSuchStreamException
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.spark.StreamingOption
+import org.apache.carbondata.spark.util.CarbonScalaUtil
+import org.apache.carbondata.streaming.CarbonStreamException
+
+/**
+ * A manager to start and stop a stream job for StreamSQL.
+ * This stream job is only available to the driver memory and not persisted
+ * so other drivers cannot see ongoing stream jobs.
+ */
+object StreamJobManager {
+ private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+
+ // map of stream name to job desc
+ private val jobs = new ConcurrentHashMap[String, StreamJobDesc]()
+
+ private def validateSourceTable(source: CarbonTable): Unit = {
+ if (!source.isStreamingSource) {
+ throw new MalformedCarbonCommandException(s"Table ${source.getTableName} is not " +
+ "streaming source table " +
+ "('streaming' tblproperty is not 'source')")
+ }
+ }
+
+ private def validateSinkTable(querySchema: StructType, sink: CarbonTable): Unit = {
+ if (!sink.isStreamingSink) {
+ throw new MalformedCarbonCommandException(s"Table ${sink.getTableName} is not " +
+ "streaming sink table " +
+ "('streaming' tblproperty is not 'sink' or 'true')")
+ }
+ val fields = sink.getCreateOrderColumn(sink.getTableName).asScala.map { column =>
+ StructField(column.getColName,
+ CarbonScalaUtil.convertCarbonToSparkDataType(column.getDataType))
+ }
+ if (!querySchema.equals(StructType(fields))) {
+ throw new MalformedCarbonCommandException(s"Schema of table ${sink.getTableName} " +
+ s"does not match query output")
+ }
+ }
+
+ /**
+ * Start a spark streaming job
+ * @param sparkSession session instance
+ * @param ifNotExists if not exists is set or not
+ * @param streamName name of the stream
+ * @param sourceTable stream source table
+ * @param sinkTable sink table to insert to
+ * @param query query string
+ * @param streamDf dataframe that containing the query from stream source table
+ * @param options options provided by user
+ * @return Job ID
+ */
+ def startStream(
+ sparkSession: SparkSession,
+ ifNotExists: Boolean,
+ streamName: String,
+ sourceTable: CarbonTable,
+ sinkTable: CarbonTable,
+ query: String,
+ streamDf: DataFrame,
+ options: StreamingOption): String = {
+ val latch = new CountDownLatch(1)
+ var exception: Throwable = null
+ var job: StreamingQuery = null
+
+ if (jobs.containsKey(streamName)) {
+ if (ifNotExists) {
+ return jobs.get(streamName).streamingQuery.id.toString
+ } else {
+ throw new MalformedCarbonCommandException(s"Stream Name $streamName already exists")
+ }
+ }
+
+ validateSourceTable(sourceTable)
+ validateSinkTable(streamDf.schema, sinkTable)
+
+ // start a new thread to run the streaming ingest job, the job will be running
+ // until user stops it by STOP STREAM JOB
+ val thread = new Thread(new Runnable {
+ override def run(): Unit = {
+ try {
+ job = streamDf.writeStream
+ .format("carbondata")
+ .trigger(options.trigger)
+ .option("checkpointLocation", options.checkpointLocation(sinkTable.getTablePath))
+ .option("dateformat", options.dateFormat)
+ .option("timestampformat", options.timeStampFormat)
+ .option("carbon.stream.parser", options.rowParser)
+ .option("dbName", sinkTable.getDatabaseName)
+ .option("tableName", sinkTable.getTableName)
+ .options(options.remainingOption)
+ .start()
+ latch.countDown()
+ job.awaitTermination()
+ } catch {
+ case e: Throwable =>
+ LOGGER.error(e)
+ exception = e
+ latch.countDown()
+ }
+ }
+ })
+ thread.start()
+
+ // wait for max 10 seconds for the streaming job to start
+ if (latch.await(10, TimeUnit.SECONDS)) {
+ if (exception != null) {
+ throw exception
+ }
+
+ jobs.put(
+ streamName,
+ StreamJobDesc(job, streamName, sourceTable.getDatabaseName, sourceTable.getTableName,
+ sinkTable.getDatabaseName, sinkTable.getTableName, query, thread))
+
+ LOGGER.audit(s"STREAM $streamName started with job id '${job.id.toString}', " +
+ s"from ${sourceTable.getDatabaseName}.${sourceTable.getTableName} " +
+ s"to ${sinkTable.getDatabaseName}.${sinkTable.getTableName}")
+ job.id.toString
+ } else {
+ thread.interrupt()
+ throw new CarbonStreamException("Streaming job takes too long to start")
+ }
+ }
+
+ /**
+ * Stop a streaming job
+ * @param streamName name of the stream
+ * @param ifExists if exists is set or not
+ */
+ def stopStream(streamName: String, ifExists: Boolean): Unit = {
+ if (jobs.containsKey(streamName)) {
+ val jobDesc = jobs.get(streamName)
+ jobDesc.streamingQuery.stop()
+ jobDesc.thread.interrupt()
+ jobs.remove(streamName)
+ LOGGER.audit(s"STREAM $streamName stopped, job id '${jobDesc.streamingQuery.id.toString}', " +
+ s"from ${jobDesc.sourceDb}.${jobDesc.sourceTable} " +
+ s"to ${jobDesc.sinkDb}.${jobDesc.sinkTable}")
+ } else {
+ if (!ifExists) {
+ throw new NoSuchStreamException(streamName)
+ }
+ }
+ }
+
+ /**
+ * Return all running jobs
+ * @return running jobs
+ */
+ def getAllJobs: Set[StreamJobDesc] = jobs.values.asScala.toSet
+
+}
+
+/**
+ * A job description for the StreamSQL job
+ */
+private[stream] case class StreamJobDesc(
+ streamingQuery: StreamingQuery,
+ streamName: String,
+ sourceDb: String,
+ sourceTable: String,
+ sinkDb: String,
+ sinkTable: String,
+ query: String,
+ thread: Thread,
+ startTime: Long = System.currentTimeMillis()
+)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
index 0a23d06..0d13d4c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
@@ -233,7 +233,7 @@ class CarbonSource extends CreatableRelationProvider with RelationProvider
}
val sparkSession = sqlContext.sparkSession
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
- if (!carbonTable.isStreamingTable) {
+ if (!carbonTable.isStreamingSink) {
throw new CarbonStreamException(s"Table ${carbonTable.getDatabaseName}." +
s"${carbonTable.getTableName} is not a streaming table")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSparkStreamingFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSparkStreamingFactory.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSparkStreamingFactory.scala
index 15b038b..cedb381 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSparkStreamingFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSparkStreamingFactory.scala
@@ -47,7 +47,7 @@ object CarbonSparkStreamingFactory {
}
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName),
tableName)(spark)
- if (!carbonTable.isStreamingTable) {
+ if (!carbonTable.isStreamingSink) {
throw new CarbonStreamException(s"Table ${carbonTable.getDatabaseName}." +
s"${carbonTable.getTableName} is not a streaming table")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index 25589d4..1ae872a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -73,7 +73,7 @@ case class CarbonCreateDataMapCommand(
}
if (mainTable != null &&
- mainTable.isStreamingTable &&
+ mainTable.isStreamingSink &&
!(dmProviderName.equalsIgnoreCase(DataMapClassProvider.PREAGGREGATE.toString)
|| dmProviderName.equalsIgnoreCase(DataMapClassProvider.TIMESERIES.toString))) {
throw new MalformedCarbonCommandException(s"Streaming table does not support creating " +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonCreateStreamCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonCreateStreamCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonCreateStreamCommand.scala
new file mode 100644
index 0000000..d3b178c
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonCreateStreamCommand.scala
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.stream
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.execution.command.DataCommand
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.execution.streaming.StreamingRelation
+import org.apache.spark.sql.types.{StringType, StructType}
+
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.spark.StreamingOption
+import org.apache.carbondata.spark.util.SparkDataTypeConverterImpl
+import org.apache.carbondata.stream.StreamJobManager
+
+/**
+ * This command will start a Spark streaming job to insert rows from source to sink
+ */
+case class CarbonCreateStreamCommand(
+ streamName: String,
+ sinkDbName: Option[String],
+ sinkTableName: String,
+ ifNotExists: Boolean,
+ optionMap: Map[String, String],
+ query: String
+) extends DataCommand {
+
+ override def output: Seq[Attribute] =
+ Seq(AttributeReference("Stream Name", StringType, nullable = false)(),
+ AttributeReference("JobId", StringType, nullable = false)(),
+ AttributeReference("Status", StringType, nullable = false)())
+
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
+ val df = sparkSession.sql(query)
+ var sourceTable: CarbonTable = null
+
+ // find the streaming source table in the query
+ // and replace it with StreamingRelation
+ val streamLp = df.logicalPlan transform {
+ case r: LogicalRelation
+ if r.relation.isInstanceOf[CarbonDatasourceHadoopRelation] &&
+ r.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonTable.isStreamingSource =>
+ val (source, streamingRelation) = prepareStreamingRelation(sparkSession, r)
+ if (sourceTable != null && sourceTable.getTableName != source.getTableName) {
+ throw new MalformedCarbonCommandException(
+ "Stream query on more than one stream source table is not supported")
+ }
+ sourceTable = source
+ streamingRelation
+ case plan: LogicalPlan => plan
+ }
+
+ if (sourceTable == null) {
+ throw new MalformedCarbonCommandException("Must specify stream source table in the query")
+ }
+
+ // start the streaming job
+ val jobId = StreamJobManager.startStream(
+ sparkSession = sparkSession,
+ ifNotExists = ifNotExists,
+ streamName = streamName,
+ sourceTable = sourceTable,
+ sinkTable = CarbonEnv.getCarbonTable(sinkDbName, sinkTableName)(sparkSession),
+ query = query,
+ streamDf = Dataset.ofRows(sparkSession, streamLp),
+ options = new StreamingOption(optionMap)
+ )
+ Seq(Row(streamName, jobId, "RUNNING"))
+ }
+
+ private def prepareStreamingRelation(
+ sparkSession: SparkSession,
+ r: LogicalRelation): (CarbonTable, StreamingRelation) = {
+ val sourceTable = r.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonTable
+ val tblProperty = sourceTable.getTableInfo.getFactTable.getTableProperties
+ val format = tblProperty.get("format")
+ if (format == null) {
+ throw new MalformedCarbonCommandException("Streaming from carbon file is not supported")
+ }
+ val streamReader = sparkSession.readStream
+ .schema(getSparkSchema(sourceTable))
+ .format(format)
+ val dataFrame = format match {
+ case "csv" | "text" | "json" | "parquet" =>
+ if (!tblProperty.containsKey("path")) {
+ throw new MalformedCarbonCommandException(
+ s"'path' tblproperty should be provided for '$format' format")
+ }
+ streamReader.load(tblProperty.get("path"))
+ case "kafka" | "socket" =>
+ streamReader.load()
+ case other =>
+ throw new MalformedCarbonCommandException(s"Streaming from $format is not supported")
+ }
+ val streamRelation = dataFrame.logicalPlan.asInstanceOf[StreamingRelation]
+
+ // Since SparkSQL analyzer will match the UUID in attribute,
+ // create a new StreamRelation and re-use the same attribute from LogicalRelation
+ (sourceTable,
+ StreamingRelation(streamRelation.dataSource, streamRelation.sourceName, r.output))
+ }
+
+ private def getSparkSchema(sourceTable: CarbonTable): StructType = {
+ val cols = sourceTable.getTableInfo.getFactTable.getListOfColumns.asScala.toArray
+ val sortedCols = cols.filter(_.getSchemaOrdinal != -1)
+ .sortWith(_.getSchemaOrdinal < _.getSchemaOrdinal)
+ SparkDataTypeConverterImpl.convertToSparkSchema(sourceTable, sortedCols)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonDropStreamCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonDropStreamCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonDropStreamCommand.scala
new file mode 100644
index 0000000..82b84ef
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonDropStreamCommand.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.stream
+
+import org.apache.spark.sql.{Row, SparkSession}
+import org.apache.spark.sql.execution.command.MetadataCommand
+
+import org.apache.carbondata.stream.StreamJobManager
+
+/**
+ * Stop the stream for specified sink table
+ */
+case class CarbonDropStreamCommand(
+ streamName: String,
+ ifExists: Boolean
+) extends MetadataCommand {
+ override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
+ StreamJobManager.stopStream(streamName, ifExists)
+ Seq.empty
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonShowStreamsCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonShowStreamsCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonShowStreamsCommand.scala
new file mode 100644
index 0000000..49c2ffb
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonShowStreamsCommand.scala
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.stream
+
+import java.util.Date
+import java.util.concurrent.TimeUnit
+
+import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
+import org.apache.spark.sql.execution.command.MetadataCommand
+import org.apache.spark.sql.types.StringType
+
+import org.apache.carbondata.stream.StreamJobManager
+
+/**
+ * Show all streams created or on a specified table
+ */
+case class CarbonShowStreamsCommand(
+ tableOp: Option[TableIdentifier]
+) extends MetadataCommand {
+ override def output: Seq[Attribute] = {
+ Seq(AttributeReference("Stream Name", StringType, nullable = false)(),
+ AttributeReference("JobId", StringType, nullable = false)(),
+ AttributeReference("Status", StringType, nullable = false)(),
+ AttributeReference("Source", StringType, nullable = false)(),
+ AttributeReference("Sink", StringType, nullable = false)(),
+ AttributeReference("Start Time", StringType, nullable = false)(),
+ AttributeReference("Time Elapse", StringType, nullable = false)())
+ }
+
+ override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
+ val jobs = tableOp match {
+ case None => StreamJobManager.getAllJobs.toSeq
+ case Some(table) =>
+ val carbonTable = CarbonEnv.getCarbonTable(table.database, table.table)(sparkSession)
+ StreamJobManager.getAllJobs.filter { job =>
+ job.sinkTable.equalsIgnoreCase(carbonTable.getTableName) &&
+ job.sinkDb.equalsIgnoreCase(carbonTable.getDatabaseName)
+ }.toSeq
+ }
+
+ jobs.map { job =>
+ val elapsedTime = System.currentTimeMillis() - job.startTime
+ Row(
+ job.streamName,
+ job.streamingQuery.id.toString,
+ if (job.streamingQuery.isActive) "RUNNING" else "FAILED",
+ s"${ job.sourceDb }.${ job.sourceTable }",
+ s"${ job.sinkDb }.${ job.sinkTable }",
+ new Date(job.startTime).toString,
+ String.format(
+ "%s days, %s hours, %s min, %s sec",
+ TimeUnit.MILLISECONDS.toDays(elapsedTime).toString,
+ TimeUnit.MILLISECONDS.toHours(elapsedTime).toString,
+ TimeUnit.MILLISECONDS.toMinutes(elapsedTime).toString,
+ TimeUnit.MILLISECONDS.toSeconds(elapsedTime).toString)
+ )
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
index 9576fb1..776750b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
@@ -71,7 +71,7 @@ case class CarbonDropTableCommand(
throw new ConcurrentOperationException(carbonTable, "loading", "drop table")
}
LOGGER.audit(s"Deleting table [$tableName] under database [$dbName]")
- if (carbonTable.isStreamingTable) {
+ if (carbonTable.isStreamingSink) {
// streaming table should acquire streaming.lock
carbonLocks += CarbonLockUtil.getLockObject(identifier, LockUsage.STREAMING_LOCK)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
index df4c742..30db50a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
@@ -100,7 +100,7 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
val updateDeltaMetadata = segmentUpdateStatusManager.readLoadMetadata()
if (updateDeltaMetadata != null && updateDeltaMetadata.nonEmpty) {
false
- } else if (relation.carbonTable.isStreamingTable) {
+ } else if (relation.carbonTable.isStreamingSink) {
false
} else {
true
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index 468121b..f5c5188 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -274,7 +274,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
// TODO remove this limitation later
val property = properties.find(_._1.equalsIgnoreCase("streaming"))
if (property.isDefined) {
- if (carbonTable.isStreamingTable) {
+ if (carbonTable.isStreamingSink) {
throw new MalformedCarbonCommandException(
"Streaming property can not be changed once it is 'true'")
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
index f9c6c5f..f4240e4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
@@ -75,7 +75,7 @@ private[sql] class StreamingTableStrategy(sparkSession: SparkSession) extends Sp
try {
streaming = CarbonEnv.getCarbonTable(
tableIdentifier.database, tableIdentifier.table)(sparkSession)
- .isStreamingTable
+ .isStreamingSink
} catch {
case e: Exception =>
streaming = false
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
index 1bb328c..c59246d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
@@ -664,7 +664,7 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
aggregationDataMapSchema: DataMapSchema,
factAggPlan: LogicalPlan): LogicalPlan = {
// to handle streaming table with pre aggregate
- if (carbonTable.isStreamingTable) {
+ if (carbonTable.isStreamingSink) {
setSegmentsForStreaming(carbonTable, aggregationDataMapSchema)
// get new fact expression
val factExp = updateFactTablePlanForStreaming(factAggPlan)
@@ -1399,11 +1399,11 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
parentTable,
parentLogicalPlan,
aggExpColumnMapping.get,
- parentTable.isStreamingTable)
+ parentTable.isStreamingSink)
} else {
Seq(attr)
}
- if(!parentTable.isStreamingTable) {
+ if(!parentTable.isStreamingSink) {
// for normal table
// generate new expression id for child
val newExpressionId = NamedExpression.newExprId
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
index 2f2048d..80d850b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
@@ -27,15 +27,13 @@ import org.apache.spark.sql.catalyst.expressions.AttributeReference
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan}
import org.apache.spark.sql.types._
import org.apache.spark.sql.util.CarbonException
-import org.apache.spark.util.SparkTypeConverter
+import org.apache.spark.util.{CarbonMetastoreTypes, SparkTypeConverter}
-import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
+import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.core.util.path.CarbonTablePath
/**
@@ -100,7 +98,7 @@ case class CarbonRelation(
val columns = carbonTable.getCreateOrderColumn(carbonTable.getTableName)
.asScala
// convert each column to Attribute
- columns.filter(!_.isInvisible).map { column =>
+ columns.filter(!_.isInvisible).map { column: CarbonColumn =>
if (column.isDimension()) {
val output: DataType = column.getDataType.getName.toLowerCase match {
case "array" =>
@@ -197,84 +195,3 @@ case class CarbonRelation(
}
}
-
-object CarbonMetastoreTypes extends RegexParsers {
- protected lazy val primitiveType: Parser[DataType] =
- "string" ^^^ StringType |
- "varchar" ^^^ StringType |
- "float" ^^^ FloatType |
- "int" ^^^ IntegerType |
- "tinyint" ^^^ ShortType |
- "short" ^^^ ShortType |
- "double" ^^^ DoubleType |
- "long" ^^^ LongType |
- "binary" ^^^ BinaryType |
- "boolean" ^^^ BooleanType |
- fixedDecimalType |
- "decimal" ^^^ "decimal" ^^^ DecimalType(10, 0) |
- "varchar\\((\\d+)\\)".r ^^^ StringType |
- "date" ^^^ DateType |
- "timestamp" ^^^ TimestampType
-
- protected lazy val fixedDecimalType: Parser[DataType] =
- "decimal" ~> "(" ~> "^[1-9]\\d*".r ~ ("," ~> "^[0-9]\\d*".r <~ ")") ^^ {
- case precision ~ scale =>
- DecimalType(precision.toInt, scale.toInt)
- }
-
- protected lazy val arrayType: Parser[DataType] =
- "array" ~> "<" ~> dataType <~ ">" ^^ {
- case tpe => ArrayType(tpe)
- }
-
- protected lazy val mapType: Parser[DataType] =
- "map" ~> "<" ~> dataType ~ "," ~ dataType <~ ">" ^^ {
- case t1 ~ _ ~ t2 => MapType(t1, t2)
- }
-
- protected lazy val structField: Parser[StructField] =
- "[a-zA-Z0-9_]*".r ~ ":" ~ dataType ^^ {
- case name ~ _ ~ tpe => StructField(name, tpe, nullable = true)
- }
-
- protected lazy val structType: Parser[DataType] =
- "struct" ~> "<" ~> repsep(structField, ",") <~ ">" ^^ {
- case fields => StructType(fields)
- }
-
- protected lazy val dataType: Parser[DataType] =
- arrayType |
- mapType |
- structType |
- primitiveType
-
- def toDataType(metastoreType: String): DataType = {
- parseAll(dataType, metastoreType) match {
- case Success(result, _) => result
- case _: NoSuccess =>
- CarbonException.analysisException(s"Unsupported dataType: $metastoreType")
- }
- }
-
- def toMetastoreType(dt: DataType): String = {
- dt match {
- case ArrayType(elementType, _) => s"array<${ toMetastoreType(elementType) }>"
- case StructType(fields) =>
- s"struct<${
- fields.map(f => s"${ f.name }:${ toMetastoreType(f.dataType) }")
- .mkString(",")
- }>"
- case StringType => "string"
- case FloatType => "float"
- case IntegerType => "int"
- case ShortType => "tinyint"
- case DoubleType => "double"
- case LongType => "bigint"
- case BinaryType => "binary"
- case BooleanType => "boolean"
- case DecimalType() => "decimal"
- case TimestampType => "timestamp"
- case DateType => "date"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
index 9dd8105..f00fcf8 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.execution.command.table.CarbonCreateTableCommand
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.CarbonExpressions.CarbonUnresolvedRelation
import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
+import org.apache.spark.sql.execution.command.stream.{CarbonCreateStreamCommand, CarbonDropStreamCommand, CarbonShowStreamsCommand}
import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.CarbonReflectionUtils
@@ -75,7 +76,7 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
protected lazy val startCommand: Parser[LogicalPlan] =
loadManagement | showLoads | alterTable | restructure | updateTable | deleteRecords |
- alterPartition | datamapManagement | alterTableFinishStreaming
+ alterPartition | datamapManagement | alterTableFinishStreaming | stream
protected lazy val loadManagement: Parser[LogicalPlan] =
deleteLoadsByID | deleteLoadsByLoadDate | cleanFiles | loadDataNew
@@ -89,6 +90,9 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
protected lazy val datamapManagement: Parser[LogicalPlan] =
createDataMap | dropDataMap | showDataMap | refreshDataMap
+ protected lazy val stream: Parser[LogicalPlan] =
+ createStream | dropStream | showStreams
+
protected lazy val alterAddPartition: Parser[LogicalPlan] =
ALTER ~> TABLE ~> (ident <~ ".").? ~ ident ~ (ADD ~> PARTITION ~>
"(" ~> repsep(stringLit, ",") <~ ")") <~ opt(";") ^^ {
@@ -146,6 +150,43 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
}
/**
+ * The syntax of CREATE STREAM
+ * CREATE STREAM [IF NOT EXISTS] streamName ON TABLE [dbName.]tableName
+ * [STMPROPERTIES('KEY'='VALUE')]
+ * AS SELECT COUNT(COL1) FROM tableName
+ */
+ protected lazy val createStream: Parser[LogicalPlan] =
+ CREATE ~> STREAM ~> opt(IF ~> NOT ~> EXISTS) ~ ident ~
+ (ON ~> TABLE ~> (ident <~ ".").?) ~ ident ~
+ (STMPROPERTIES ~> "(" ~> repsep(loadOptions, ",") <~ ")").? ~
+ (AS ~> restInput) <~ opt(";") ^^ {
+ case ifNotExists ~ streamName ~ dbName ~ tableName ~ options ~ query =>
+ val optionMap = options.getOrElse(List[(String, String)]()).toMap[String, String]
+ CarbonCreateStreamCommand(
+ streamName, dbName, tableName, ifNotExists.isDefined, optionMap, query)
+ }
+
+ /**
+ * The syntax of DROP STREAM
+ * DROP STREAM [IF EXISTS] streamName
+ */
+ protected lazy val dropStream: Parser[LogicalPlan] =
+ DROP ~> STREAM ~> opt(IF ~> EXISTS) ~ ident <~ opt(";") ^^ {
+ case ifExists ~ streamName =>
+ CarbonDropStreamCommand(streamName, ifExists.isDefined)
+ }
+
+ /**
+ * The syntax of SHOW STREAMS
+ * SHOW STREAMS [ON TABLE dbName.tableName]
+ */
+ protected lazy val showStreams: Parser[LogicalPlan] =
+ SHOW ~> STREAMS ~> opt(ontable) <~ opt(";") ^^ {
+ case tableIdent =>
+ CarbonShowStreamsCommand(tableIdent)
+ }
+
+ /**
* The syntax of datamap creation is as follows.
* CREATE DATAMAP IF NOT EXISTS datamapName [ON TABLE tableName]
* USING 'DataMapProviderName'
@@ -160,7 +201,6 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
(DMPROPERTIES ~> "(" ~> repsep(loadOptions, ",") <~ ")").? ~
(AS ~> restInput).? <~ opt(";") ^^ {
case ifnotexists ~ dmname ~ tableIdent ~ dmProviderName ~ deferred ~ dmprops ~ query =>
-
val map = dmprops.getOrElse(List[(String, String)]()).toMap[String, String]
CarbonCreateDataMapCommand(dmname, tableIdent, dmProviderName, map, query,
ifnotexists.isDefined, deferred.isDefined)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index b7b28b8..066819e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -120,8 +120,8 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf,
def needToConvertToLowerCase(key: String): Boolean = {
- val noConvertList = Array("LIST_INFO", "RANGE_INFO")
- !noConvertList.exists(x => x.equalsIgnoreCase(key));
+ val noConvertList = Array("LIST_INFO", "RANGE_INFO", "PATH")
+ !noConvertList.exists(x => x.equalsIgnoreCase(key))
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/main/scala/org/apache/spark/util/SparkTypeConverter.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/SparkTypeConverter.scala b/integration/spark2/src/main/scala/org/apache/spark/util/SparkTypeConverter.scala
deleted file mode 100644
index 65210b8..0000000
--- a/integration/spark2/src/main/scala/org/apache/spark/util/SparkTypeConverter.scala
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.util
-
-import java.util.Objects
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.hive.CarbonMetastoreTypes
-import org.apache.spark.sql.types
-import org.apache.spark.sql.types._
-
-import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension, ColumnSchema}
-
-private[spark] object SparkTypeConverter {
-
- def createSparkSchema(table: CarbonTable, columns: Seq[String]): StructType = {
- Objects.requireNonNull(table)
- Objects.requireNonNull(columns)
- if (columns.isEmpty) {
- throw new IllegalArgumentException("column list is empty")
- }
- val fields = new java.util.ArrayList[StructField](columns.size)
- val allColumns = table.getTableInfo.getFactTable.getListOfColumns.asScala
-
- // find the column and add it to fields array
- columns.foreach { column =>
- val col = allColumns.find(_.getColumnName.equalsIgnoreCase(column)).getOrElse(
- throw new IllegalArgumentException(column + " does not exist")
- )
- fields.add(StructField(col.getColumnName, convertCarbonToSparkDataType(col, table)))
- }
- StructType(fields)
- }
-
- /**
- * Converts from carbon datatype to corresponding spark datatype.
- */
- def convertCarbonToSparkDataType(
- columnSchema: ColumnSchema,
- table: CarbonTable): types.DataType = {
- if (CarbonDataTypes.isDecimal(columnSchema.getDataType)) {
- val scale = columnSchema.getScale
- val precision = columnSchema.getPrecision
- if (scale == 0 && precision == 0) {
- DecimalType(18, 2)
- } else {
- DecimalType(precision, scale)
- }
- } else if (CarbonDataTypes.isArrayType(columnSchema.getDataType)) {
- CarbonMetastoreTypes
- .toDataType(s"array<${ getArrayChildren(table, columnSchema.getColumnName) }>")
- } else if (CarbonDataTypes.isStructType(columnSchema.getDataType)) {
- CarbonMetastoreTypes
- .toDataType(s"struct<${ getStructChildren(table, columnSchema.getColumnName) }>")
- } else {
- columnSchema.getDataType match {
- case CarbonDataTypes.STRING => StringType
- case CarbonDataTypes.SHORT => ShortType
- case CarbonDataTypes.INT => IntegerType
- case CarbonDataTypes.LONG => LongType
- case CarbonDataTypes.DOUBLE => DoubleType
- case CarbonDataTypes.BOOLEAN => BooleanType
- case CarbonDataTypes.TIMESTAMP => TimestampType
- case CarbonDataTypes.DATE => DateType
- }
- }
- }
-
- def getArrayChildren(table: CarbonTable, dimName: String): String = {
- table.getChildren(dimName).asScala.map(childDim => {
- childDim.getDataType.getName.toLowerCase match {
- case "array" => s"array<${ getArrayChildren(table, childDim.getColName) }>"
- case "struct" => s"struct<${ getStructChildren(table, childDim.getColName) }>"
- case dType => addDecimalScaleAndPrecision(childDim, dType)
- }
- }).mkString(",")
- }
-
- def getStructChildren(table: CarbonTable, dimName: String): String = {
- table.getChildren(dimName).asScala.map(childDim => {
- childDim.getDataType.getName.toLowerCase match {
- case "array" => s"${
- childDim.getColName.substring(dimName.length + 1)
- }:array<${ getArrayChildren(table, childDim.getColName) }>"
- case "struct" => s"${
- childDim.getColName.substring(dimName.length + 1)
- }:struct<${ table.getChildren(childDim.getColName)
- .asScala.map(f => s"${ recursiveMethod(table, childDim.getColName, f) }").mkString(",")
- }>"
- case dType => s"${ childDim.getColName
- .substring(dimName.length() + 1) }:${ addDecimalScaleAndPrecision(childDim, dType) }"
- }
- }).mkString(",")
- }
-
- def addDecimalScaleAndPrecision(dimval: CarbonColumn, dataType: String): String = {
- var dType = dataType
- if (CarbonDataTypes.isDecimal(dimval.getDataType)) {
- dType +=
- "(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema.getScale + ")"
- }
- dType
- }
-
- private def recursiveMethod(
- table: CarbonTable, dimName: String, childDim: CarbonDimension) = {
- childDim.getDataType.getName.toLowerCase match {
- case "array" => s"${
- childDim.getColName.substring(dimName.length + 1)
- }:array<${ getArrayChildren(table, childDim.getColName) }>"
- case "struct" => s"${
- childDim.getColName.substring(dimName.length + 1)
- }:struct<${ getStructChildren(table, childDim.getColName) }>"
- case dType => s"${ childDim.getColName.substring(dimName.length + 1) }:${ dType }"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index 3253c3d..9d9a9f5 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -25,13 +25,15 @@ import java.util.concurrent.Executors
import scala.collection.mutable
+import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.hive.CarbonRelation
-import org.apache.spark.sql.{CarbonEnv, Row, SaveMode, SparkSession}
import org.apache.spark.sql.streaming.{ProcessingTime, StreamingQuery}
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.exceptions.NoSuchStreamException
import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.TIMESERIES
@@ -120,7 +122,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
createTable(tableName = "agg_table", streaming = true, withBatchLoad = false)
- val csvDataDir = new File("target/csvdatanew").getCanonicalPath
+ var csvDataDir = integrationPath + "/spark2/target/csvdatanew"
generateCSVDataFile(spark, idStart = 10, rowNums = 5, csvDataDir)
generateCSVDataFile(spark, idStart = 10, rowNums = 5, csvDataDir, SaveMode.Append)
}
@@ -188,6 +190,10 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
dropTable()
sql("USE default")
sql("DROP DATABASE IF EXISTS streaming CASCADE")
+ var csvDataDir = integrationPath + "/spark2/target/csvdatanew"
+ new File(csvDataDir).delete()
+ csvDataDir = integrationPath + "/spark2/target/csvdata"
+ new File(csvDataDir).delete()
}
def dropTable(): Unit = {
@@ -362,12 +368,12 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
sql("alter table agg_table2 compact 'streaming'")
// Data should be loaded into aggregate table as hand-off is fired
checkAnswer(sql("select name, sum(salary) from agg_table2 group by name"),
- Seq(
- Row("name_10", 400000.0),
- Row("name_14", 560000.0),
- Row("name_12", 480000.0),
- Row("name_11", 440000.0),
- Row("name_13", 520000.0)))
+ Seq(
+ Row("name_10", 400000.0),
+ Row("name_14", 560000.0),
+ Row("name_12", 480000.0),
+ Row("name_11", 440000.0),
+ Row("name_13", 520000.0)))
checkAnswer(sql("select * from agg_table2_p1"),
Seq(
Row("name_10", 200000.0),
@@ -1430,7 +1436,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
)
val table1 =
CarbonEnv.getCarbonTable(Option("streaming"), "stream_table_reopen")(spark)
- assertResult(true)(table1.isStreamingTable)
+ assertResult(true)(table1.isStreamingSink)
sql("alter table streaming.stream_table_reopen compact 'close_streaming'")
@@ -1447,13 +1453,13 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
val table2 =
CarbonEnv.getCarbonTable(Option("streaming"), "stream_table_reopen")(spark)
- assertResult(false)(table2.isStreamingTable)
+ assertResult(false)(table2.isStreamingSink)
sql("ALTER TABLE streaming.stream_table_reopen SET TBLPROPERTIES('streaming'='true')")
val table3 =
CarbonEnv.getCarbonTable(Option("streaming"), "stream_table_reopen")(spark)
- assertResult(true)(table3.isStreamingTable)
+ assertResult(true)(table3.isStreamingSink)
executeStreamingIngest(
tableName = "stream_table_reopen",
@@ -1569,8 +1575,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
test("test bad_record_action IGNORE on streaming table") {
-
-sql("drop table if exists streaming.bad_record_ignore")
+ sql("drop table if exists streaming.bad_record_ignore")
sql(
s"""
| CREATE TABLE streaming.bad_record_ignore(
@@ -1628,6 +1633,334 @@ sql("drop table if exists streaming.bad_record_ignore")
checkAnswer(sql("select count(*) from streaming.bad_record_redirect"), Seq(Row(19)))
}
+ test("StreamSQL: create and drop a stream") {
+ sql("DROP TABLE IF EXISTS source")
+ sql("DROP TABLE IF EXISTS sink")
+
+ var rows = sql("SHOW STREAMS").collect()
+ assertResult(0)(rows.length)
+
+ val csvDataDir = integrationPath + "/spark2/target/streamSql"
+ // streaming ingest 10 rows
+ generateCSVDataFile(spark, idStart = 10, rowNums = 10, csvDataDir)
+
+ sql(
+ s"""
+ |CREATE TABLE source(
+ | id INT,
+ | name STRING,
+ | city STRING,
+ | salary FLOAT,
+ | tax DECIMAL(8,2),
+ | percent double,
+ | birthday DATE,
+ | register TIMESTAMP,
+ | updated TIMESTAMP
+ |)
+ |STORED AS carbondata
+ |TBLPROPERTIES (
+ | 'streaming'='source',
+ | 'format'='csv',
+ | 'path'='$csvDataDir'
+ |)
+ """.stripMargin)
+
+ sql(
+ s"""
+ |CREATE TABLE sink(
+ | id INT,
+ | name STRING,
+ | city STRING,
+ | salary FLOAT,
+ | tax DECIMAL(8,2),
+ | percent double,
+ | birthday DATE,
+ | register TIMESTAMP,
+ | updated TIMESTAMP
+ | )
+ |STORED AS carbondata
+ |TBLPROPERTIES('streaming'='sink')
+ """.stripMargin)
+
+ sql(
+ """
+ |CREATE STREAM stream123 ON TABLE sink
+ |STMPROPERTIES(
+ | 'trigger'='ProcessingTime',
+ | 'interval'='1 seconds')
+ |AS
+ | SELECT *
+ | FROM source
+ | WHERE id % 2 = 1
+ """.stripMargin).show(false)
+ sql(
+ """
+ |CREATE STREAM IF NOT EXISTS stream123 ON TABLE sink
+ |STMPROPERTIES(
+ | 'trigger'='ProcessingTime',
+ | 'interval'='1 seconds')
+ |AS
+ | SELECT *
+ | FROM source
+ | WHERE id % 2 = 1
+ """.stripMargin).show(false)
+ Thread.sleep(200)
+ sql("select * from sink").show
+
+ generateCSVDataFile(spark, idStart = 30, rowNums = 10, csvDataDir, SaveMode.Append)
+ Thread.sleep(5000)
+
+ // after 2 minibatch, there should be 10 row added (filter condition: id%2=1)
+ checkAnswer(sql("select count(*) from sink"), Seq(Row(10)))
+
+ val row = sql("select * from sink order by id").head()
+ val exceptedRow = Row(11, "name_11", "city_11", 110000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))
+ assertResult(exceptedRow)(row)
+
+ sql("SHOW STREAMS").show(false)
+
+ rows = sql("SHOW STREAMS").collect()
+ assertResult(1)(rows.length)
+ assertResult("stream123")(rows.head.getString(0))
+ assertResult("RUNNING")(rows.head.getString(2))
+ assertResult("streaming.source")(rows.head.getString(3))
+ assertResult("streaming.sink")(rows.head.getString(4))
+
+ rows = sql("SHOW STREAMS ON TABLE sink").collect()
+ assertResult(1)(rows.length)
+ assertResult("stream123")(rows.head.getString(0))
+ assertResult("RUNNING")(rows.head.getString(2))
+ assertResult("streaming.source")(rows.head.getString(3))
+ assertResult("streaming.sink")(rows.head.getString(4))
+
+ sql("DROP STREAM stream123")
+ sql("DROP STREAM IF EXISTS stream123")
+
+ rows = sql("SHOW STREAMS").collect()
+ assertResult(0)(rows.length)
+
+ sql("DROP TABLE IF EXISTS source")
+ sql("DROP TABLE IF EXISTS sink")
+ }
+
+ test("StreamSQL: create stream without interval ") {
+ sql("DROP TABLE IF EXISTS source")
+ sql("DROP TABLE IF EXISTS sink")
+
+ val csvDataDir = integrationPath + "/spark2/target/streamsql"
+ // streaming ingest 10 rows
+ generateCSVDataFile(spark, idStart = 10, rowNums = 10, csvDataDir)
+
+ sql(
+ s"""
+ |CREATE TABLE source(
+ | id INT,
+ | name STRING,
+ | city STRING,
+ | salary FLOAT,
+ | tax DECIMAL(8,2),
+ | percent double,
+ | birthday DATE,
+ | register TIMESTAMP,
+ | updated TIMESTAMP
+ |)
+ |STORED AS carbondata
+ |TBLPROPERTIES (
+ | 'streaming'='source',
+ | 'format'='csv',
+ | 'path'='$csvDataDir'
+ |)
+ """.stripMargin)
+ sql(
+ s"""
+ |CREATE TABLE sink(
+ | id INT,
+ | name STRING,
+ | city STRING,
+ | salary FLOAT,
+ | tax DECIMAL(8,2),
+ | percent double,
+ | birthday DATE,
+ | register TIMESTAMP,
+ | updated TIMESTAMP
+ | )
+ |STORED AS carbondata
+ |TBLPROPERTIES('streaming'='sink')
+ """.stripMargin)
+ val ex = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ |CREATE STREAM stream456 ON TABLE sink
+ |STMPROPERTIES(
+ | 'trigger'='ProcessingTime')
+ |AS
+ | SELECT *
+ | FROM source
+ | WHERE id % 2 = 1
+ """.stripMargin)
+ }
+ assert(ex.getMessage.contains("interval must be specified"))
+ sql("DROP TABLE IF EXISTS source")
+ sql("DROP TABLE IF EXISTS sink")
+ }
+
+ test("StreamSQL: create stream on non exist stream source table") {
+ sql("DROP TABLE IF EXISTS sink")
+ sql(
+ s"""
+ |CREATE TABLE sink(
+ | id INT,
+ | name STRING,
+ | city STRING,
+ | salary FLOAT,
+ | tax DECIMAL(8,2),
+ | percent double,
+ | birthday DATE,
+ | register TIMESTAMP,
+ | updated TIMESTAMP
+ | )
+ |STORED AS carbondata
+ |TBLPROPERTIES('streaming'='true')
+ """.stripMargin)
+
+ val ex = intercept[AnalysisException] {
+ sql(
+ """
+ |CREATE STREAM stream123 ON TABLE sink
+ |STMPROPERTIES(
+ | 'trigger'='ProcessingTime',
+ | 'interval'='1 seconds')
+ |AS
+ | SELECT *
+ | FROM source
+ | WHERE id % 2 = 1
+ """.stripMargin).show(false)
+ }
+ sql("DROP TABLE IF EXISTS sink")
+ }
+
+ test("StreamSQL: create stream source using carbon file") {
+ sql("DROP TABLE IF EXISTS source")
+ sql("DROP TABLE IF EXISTS sink")
+
+ sql(
+ s"""
+ |CREATE TABLE source(
+ | id INT,
+ | name STRING,
+ | city STRING,
+ | salary FLOAT,
+ | tax DECIMAL(8,2),
+ | percent double,
+ | birthday DATE,
+ | register TIMESTAMP,
+ | updated TIMESTAMP
+ |)
+ |STORED AS carbondata
+ |TBLPROPERTIES (
+ | 'streaming'='source'
+ |)
+ """.stripMargin)
+
+ sql(
+ s"""
+ |CREATE TABLE sink(
+ | id INT,
+ | name STRING,
+ | city STRING,
+ | salary FLOAT,
+ | tax DECIMAL(8,2),
+ | percent double,
+ | birthday DATE,
+ | register TIMESTAMP,
+ | updated TIMESTAMP
+ | )
+ |STORED AS carbondata
+ |TBLPROPERTIES('streaming'='sink')
+ """.stripMargin)
+
+ val ex = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ |CREATE STREAM stream123 ON TABLE sink
+ |STMPROPERTIES(
+ | 'trigger'='ProcessingTime',
+ | 'interval'='1 seconds')
+ |AS
+ | SELECT *
+ | FROM source
+ | WHERE id % 2 = 1
+ """.stripMargin)
+ }
+ assert(ex.getMessage.contains("Streaming from carbon file is not supported"))
+
+ sql("DROP TABLE IF EXISTS source")
+ sql("DROP TABLE IF EXISTS sink")
+ }
+
+ test("StreamSQL: start stream on non-stream table") {
+ sql(
+ s"""
+ |CREATE TABLE notsource(
+ | id INT,
+ | name STRING,
+ | city STRING,
+ | salary FLOAT,
+ | tax DECIMAL(8,2),
+ | percent double,
+ | birthday DATE,
+ | register TIMESTAMP,
+ | updated TIMESTAMP
+ | )
+ |STORED AS carbondata
+ """.stripMargin)
+ sql(
+ s"""
+ |CREATE TABLE sink(
+ | id INT,
+ | name STRING,
+ | city STRING,
+ | salary FLOAT,
+ | tax DECIMAL(8,2),
+ | percent double,
+ | birthday DATE,
+ | register TIMESTAMP,
+ | updated TIMESTAMP
+ | )
+ |STORED AS carbondata
+ |TBLPROPERTIES('streaming'='true')
+ """.stripMargin)
+
+ val ex = intercept[MalformedCarbonCommandException] {
+ sql(
+ """
+ |CREATE STREAM stream456 ON TABLE sink
+ |STMPROPERTIES(
+ | 'trigger'='ProcessingTime',
+ | 'interval'='1 seconds')
+ |AS
+ | SELECT *
+ | FROM notsource
+ | WHERE id % 2 = 1
+ """.stripMargin).show(false)
+ }
+ assert(ex.getMessage.contains("Must specify stream source table in the query"))
+ sql("DROP TABLE sink")
+ }
+
+ test("StreamSQL: drop stream on non exist table") {
+ val ex = intercept[NoSuchStreamException] {
+ sql("DROP STREAM streamyyy")
+ }
+ assert(ex.getMessage.contains("stream 'streamyyy' not found"))
+ }
+
+ test("StreamSQL: show streams on non-exist table") {
+ val ex = intercept[NoSuchTableException] {
+ sql("SHOW STREAMS ON TABLE ddd")
+ }
+ assert(ex.getMessage.contains("'ddd' not found"))
+ }
def createWriteSocketThread(
serverSocket: ServerSocket,
[10/50] [abbrv] carbondata git commit: [CARBONDATA-2554] Added
support for logical type
Posted by ja...@apache.org.
[CARBONDATA-2554] Added support for logical type
Added support for date and timestamp logical types in AvroCarbonWriter.
This closes #2347
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2f234869
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2f234869
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2f234869
Branch: refs/heads/carbonstore
Commit: 2f2348690964ac87c2f38939280958f2469d212d
Parents: 27d7059
Author: kunal642 <ku...@gmail.com>
Authored: Mon May 28 11:41:59 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Tue Jun 5 11:52:09 2018 +0530
----------------------------------------------------------------------
.../DirectDictionaryGenerator.java | 2 +
.../DateDirectDictionaryGenerator.java | 2 +-
.../TimeStampDirectDictionaryGenerator.java | 2 +-
.../TestNonTransactionalCarbonTable.scala | 145 ++++++++++++++++++-
.../processing/datatypes/PrimitiveDataType.java | 44 +++++-
.../loading/dictionary/DirectDictionary.java | 4 +
.../InputProcessorStepWithNoConverterImpl.java | 24 ++-
.../carbondata/sdk/file/AvroCarbonWriter.java | 71 ++++++++-
8 files changed, 279 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2f234869/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
index 469fe1e..2139f31 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
@@ -40,6 +40,8 @@ public interface DirectDictionaryGenerator {
*/
Object getValueFromSurrogate(int key);
+ int generateKey(long value);
+
/**
* The method generate and returns the dictionary / surrogate key for direct dictionary column
* This Method is called while executing filter queries for getting direct surrogate members.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2f234869/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index c49af9c..329e260 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -163,7 +163,7 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
}
}
- private int generateKey(long timeValue) {
+ public int generateKey(long timeValue) {
if (timeValue < MIN_VALUE || timeValue > MAX_VALUE) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Value for date type column is not in valid range. Value considered as null.");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2f234869/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index d218e99..c7a4194 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -206,7 +206,7 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
}
}
- private int generateKey(long timeValue) {
+ public int generateKey(long timeValue) {
long time = (timeValue - cutOffTimeStamp) / granularityFactor;
int keyValue = -1;
if (time >= (long) Integer.MIN_VALUE && time <= (long) Integer.MAX_VALUE) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2f234869/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index 5beb9c4..095d12d 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -17,7 +17,7 @@
package org.apache.carbondata.spark.testsuite.createTable
-import java.sql.Timestamp
+import java.sql.{Date, Timestamp}
import java.io.{File, FileFilter, IOException}
import java.util
import java.util.concurrent.TimeUnit
@@ -42,6 +42,7 @@ import scala.concurrent.duration.Duration
import org.apache.avro
import org.apache.commons.lang.CharEncoding
+import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
import tech.allegro.schema.json2avro.converter.JsonAvroConverter
import org.apache.carbondata.core.metadata.datatype.{DataTypes, StructField}
@@ -2151,4 +2152,146 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
writer.close()
}
+ test("test logical type date") {
+ sql("drop table if exists sdkOutputTable")
+ FileFactory.deleteAllCarbonFilesOfDir(FileFactory.getCarbonFile(writerPath))
+ val schema1 =
+ """{
+ | "namespace": "com.apache.schema",
+ | "type": "record",
+ | "name": "StudentActivity",
+ | "fields": [
+ | {
+ | "name": "id",
+ | "type": {"type" : "int", "logicalType": "date"}
+ | },
+ | {
+ | "name": "course_details",
+ | "type": {
+ | "name": "course_details",
+ | "type": "record",
+ | "fields": [
+ | {
+ | "name": "course_struct_course_time",
+ | "type": {"type" : "int", "logicalType": "date"}
+ | }
+ | ]
+ | }
+ | }
+ | ]
+ |}""".stripMargin
+
+ val json1 =
+ """{"id": 101, "course_details": { "course_struct_course_time":10}}""".stripMargin
+ val nn = new org.apache.avro.Schema.Parser().parse(schema1)
+ val converter = new JsonAvroConverter
+ val record = converter
+ .convertToGenericDataRecord(json1.getBytes(CharEncoding.UTF_8), nn)
+
+ val writer = CarbonWriter.builder
+ .outputPath(writerPath).isTransactionalTable(false).buildWriterForAvroInput(nn)
+ writer.write(record)
+ writer.close()
+ sql(
+ s"""CREATE EXTERNAL TABLE sdkOutputTable(dateType date, course_details struct<course_struct_course_time: date>) STORED BY
+ |'carbondata' LOCATION
+ |'$writerPath' """.stripMargin)
+ checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(java.sql.Date.valueOf("1970-04-12"), Row(java.sql.Date.valueOf("1970-01-11")))))
+ }
+
+ test("test logical type timestamp-millis") {
+ sql("drop table if exists sdkOutputTable")
+ FileFactory.deleteAllCarbonFilesOfDir(FileFactory.getCarbonFile(writerPath))
+ val schema1 =
+ """{
+ | "namespace": "com.apache.schema",
+ | "type": "record",
+ | "name": "StudentActivity",
+ | "fields": [
+ | {
+ | "name": "id",
+ | "type": {"type" : "long", "logicalType": "timestamp-millis"}
+ | },
+ | {
+ | "name": "course_details",
+ | "type": {
+ | "name": "course_details",
+ | "type": "record",
+ | "fields": [
+ | {
+ | "name": "course_struct_course_time",
+ | "type": {"type" : "long", "logicalType": "timestamp-millis"}
+ | }
+ | ]
+ | }
+ | }
+ | ]
+ |}""".stripMargin
+
+ val json1 =
+ """{"id": 172800000,"course_details": { "course_struct_course_time":172800000}}""".stripMargin
+
+ val nn = new org.apache.avro.Schema.Parser().parse(schema1)
+ val converter = new JsonAvroConverter
+ val record = converter
+ .convertToGenericDataRecord(json1.getBytes(CharEncoding.UTF_8), nn)
+
+ val writer = CarbonWriter.builder
+ .outputPath(writerPath).isTransactionalTable(false).buildWriterForAvroInput(nn)
+ writer.write(record)
+ writer.close()
+ sql(
+ s"""CREATE EXTERNAL TABLE sdkOutputTable(dateType timestamp, course_details struct<course_struct_course_time: timestamp>) STORED BY
+ |'carbondata' LOCATION
+ |'$writerPath' """.stripMargin)
+ checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Timestamp.valueOf("1970-01-02 16:00:00"), Row(Timestamp.valueOf("1970-01-02 16:00:00")))))
+ }
+
+ test("test logical type-micros timestamp") {
+ sql("drop table if exists sdkOutputTable")
+ FileFactory.deleteAllCarbonFilesOfDir(FileFactory.getCarbonFile(writerPath))
+ val schema1 =
+ """{
+ | "namespace": "com.apache.schema",
+ | "type": "record",
+ | "name": "StudentActivity",
+ | "fields": [
+ | {
+ | "name": "id",
+ | "type": {"type" : "long", "logicalType": "timestamp-micros"}
+ | },
+ | {
+ | "name": "course_details",
+ | "type": {
+ | "name": "course_details",
+ | "type": "record",
+ | "fields": [
+ | {
+ | "name": "course_struct_course_time",
+ | "type": {"type" : "long", "logicalType": "timestamp-micros"}
+ | }
+ | ]
+ | }
+ | }
+ | ]
+ |}""".stripMargin
+
+ val json1 =
+ """{"id": 172800000000,"course_details": { "course_struct_course_time":172800000000}}""".stripMargin
+
+ val nn = new org.apache.avro.Schema.Parser().parse(schema1)
+ val converter = new JsonAvroConverter
+ val record = converter
+ .convertToGenericDataRecord(json1.getBytes(CharEncoding.UTF_8), nn)
+
+ val writer = CarbonWriter.builder
+ .outputPath(writerPath).isTransactionalTable(false).buildWriterForAvroInput(nn)
+ writer.write(record)
+ writer.close()
+ sql(
+ s"""CREATE EXTERNAL TABLE sdkOutputTable(dateType timestamp, course_details struct<course_struct_course_time: timestamp>) STORED BY
+ |'carbondata' LOCATION
+ |'$writerPath' """.stripMargin)
+ checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Timestamp.valueOf("1970-01-02 16:00:00"), Row(Timestamp.valueOf("1970-01-02 16:00:00")))))
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2f234869/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
index 7450b82..3a477ce 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
@@ -288,7 +288,11 @@ public class PrimitiveDataType implements GenericDataType<Object> {
logHolder.setReason(message);
}
} else {
- surrogateKey = dictionaryGenerator.getOrGenerateKey(parsedValue);
+ if (dictionaryGenerator instanceof DirectDictionary && input instanceof Long) {
+ surrogateKey = ((DirectDictionary) dictionaryGenerator).generateKey((long) input);
+ } else {
+ surrogateKey = dictionaryGenerator.getOrGenerateKey(parsedValue);
+ }
if (surrogateKey == CarbonCommonConstants.INVALID_SURROGATE_KEY) {
surrogateKey = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
message = CarbonDataProcessorUtil
@@ -316,15 +320,36 @@ public class PrimitiveDataType implements GenericDataType<Object> {
if (!this.carbonDimension.getUseActualData()) {
byte[] value = null;
if (isDirectDictionary) {
- int surrogateKey = dictionaryGenerator.getOrGenerateKey(parsedValue);
+ int surrogateKey;
+ // If the input is a long value then this means that logical type was provided by
+ // the user using AvroCarbonWriter. In this case directly generate surrogate key
+ // using dictionaryGenerator.
+ if (dictionaryGenerator instanceof DirectDictionary && input instanceof Long) {
+ surrogateKey = ((DirectDictionary) dictionaryGenerator).generateKey((long) input);
+ } else {
+ surrogateKey = dictionaryGenerator.getOrGenerateKey(parsedValue);
+ }
if (surrogateKey == CarbonCommonConstants.INVALID_SURROGATE_KEY) {
value = new byte[0];
} else {
value = ByteUtil.toBytes(surrogateKey);
}
} else {
- value = DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(parsedValue,
- this.carbonDimension.getDataType(), dateFormat);
+ // If the input is a long value then this means that logical type was provided by
+ // the user using AvroCarbonWriter. In this case directly generate Bytes from value.
+ if (this.carbonDimension.getDataType().equals(DataTypes.DATE)
+ || this.carbonDimension.getDataType().equals(DataTypes.TIMESTAMP)
+ && input instanceof Long) {
+ if (dictionaryGenerator != null) {
+ value = ByteUtil.toBytes(((DirectDictionary) dictionaryGenerator)
+ .generateKey((long) input));
+ } else {
+ value = ByteUtil.toBytes(Long.parseLong(parsedValue));
+ }
+ } else {
+ value = DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(parsedValue,
+ this.carbonDimension.getDataType(), dateFormat);
+ }
if (this.carbonDimension.getDataType() == DataTypes.STRING
&& value.length > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
throw new CarbonDataLoadingException("Dataload failed, String size cannot exceed "
@@ -333,8 +358,15 @@ public class PrimitiveDataType implements GenericDataType<Object> {
}
updateValueToByteStream(dataOutputStream, value);
} else {
- Object value = DataTypeUtil.getDataDataTypeForNoDictionaryColumn(parsedValue,
- this.carbonDimension.getDataType(), dateFormat);
+ Object value;
+ if (dictionaryGenerator instanceof DirectDictionary
+ && input instanceof Long) {
+ value = ByteUtil.toBytes(
+ ((DirectDictionary) dictionaryGenerator).generateKey((long) input));
+ } else {
+ value = DataTypeUtil.getDataDataTypeForNoDictionaryColumn(parsedValue,
+ this.carbonDimension.getDataType(), dateFormat);
+ }
if (this.carbonDimension.getDataType() == DataTypes.STRING
&& value.toString().length() > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
throw new CarbonDataLoadingException("Dataload failed, String size cannot exceed "
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2f234869/processing/src/main/java/org/apache/carbondata/processing/loading/dictionary/DirectDictionary.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/dictionary/DirectDictionary.java b/processing/src/main/java/org/apache/carbondata/processing/loading/dictionary/DirectDictionary.java
index 165e5a4..33dc8e3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/dictionary/DirectDictionary.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/dictionary/DirectDictionary.java
@@ -46,6 +46,10 @@ public class DirectDictionary implements BiDictionary<Integer, Object> {
return dictionaryGenerator.generateDirectSurrogateKey(value.toString());
}
+ public Integer generateKey(long value) {
+ return dictionaryGenerator.generateKey(value);
+ }
+
@Override
public Object getValue(Integer key) {
return dictionaryGenerator.getValueFromSurrogate(key);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2f234869/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java
index c99a413..5f7a94c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java
@@ -28,6 +28,8 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
@@ -215,6 +217,10 @@ public class InputProcessorStepWithNoConverterImpl extends AbstractDataLoadProce
private Map<Integer, GenericDataType> dataFieldsWithComplexDataType;
+ private DirectDictionaryGenerator dateDictionaryGenerator;
+
+ private DirectDictionaryGenerator timestampDictionaryGenerator;
+
public InputProcessorIterator(List<CarbonIterator<Object[]>> inputIterators, int batchSize,
boolean preFetch, AtomicLong rowCounter, int[] orderOfData, boolean[] noDictionaryMapping,
DataType[] dataTypes, CarbonDataLoadConfiguration configuration,
@@ -313,7 +319,23 @@ public class InputProcessorStepWithNoConverterImpl extends AbstractDataLoadProce
throw new CarbonDataLoadingException("Loading Exception", e);
}
} else {
- newData[i] = data[orderOfData[i]];
+ DataType dataType = dataFields[i].getColumn().getDataType();
+ if (dataType == DataTypes.DATE && data[orderOfData[i]] instanceof Long) {
+ if (dateDictionaryGenerator == null) {
+ dateDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(dataType, dataFields[i].getDateFormat());
+ }
+ newData[i] = dateDictionaryGenerator.generateKey((long) data[orderOfData[i]]);
+ } else if (dataType == DataTypes.TIMESTAMP && data[orderOfData[i]] instanceof Long) {
+ if (timestampDictionaryGenerator == null) {
+ timestampDictionaryGenerator =
+ DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(dataType, dataFields[i].getTimestampFormat());
+ }
+ newData[i] = timestampDictionaryGenerator.generateKey((long) data[orderOfData[i]]);
+ } else {
+ newData[i] = data[orderOfData[i]];
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2f234869/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
index 8bbf364..edecd6b 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
@@ -24,15 +24,21 @@ import java.util.Random;
import java.util.UUID;
import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.DateDirectDictionaryGenerator;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.datatype.StructField;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat;
import org.apache.carbondata.hadoop.internal.ObjectArrayWritable;
import org.apache.carbondata.processing.loading.complexobjects.ArrayObject;
import org.apache.carbondata.processing.loading.complexobjects.StructObject;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
+import org.apache.avro.LogicalType;
+import org.apache.avro.LogicalTypes;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.hadoop.conf.Configuration;
@@ -55,6 +61,8 @@ public class AvroCarbonWriter extends CarbonWriter {
private TaskAttemptContext context;
private ObjectArrayWritable writable;
private Schema avroSchema;
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CarbonTable.class.getName());
AvroCarbonWriter(CarbonLoadModel loadModel) throws IOException {
Configuration hadoopConf = new Configuration();
@@ -88,10 +96,35 @@ public class AvroCarbonWriter extends CarbonWriter {
private Object avroFieldToObject(Schema.Field avroField, Object fieldValue) {
Object out;
Schema.Type type = avroField.schema().getType();
+ LogicalType logicalType = avroField.schema().getLogicalType();
switch (type) {
- case BOOLEAN:
case INT:
+ if (logicalType != null) {
+ if (logicalType instanceof LogicalTypes.Date) {
+ int dateIntValue = (int) fieldValue;
+ out = dateIntValue * DateDirectDictionaryGenerator.MILLIS_PER_DAY;
+ } else {
+ LOGGER.warn("Actual type: INT, Logical Type: " + logicalType.getName());
+ out = fieldValue;
+ }
+ } else {
+ out = fieldValue;
+ }
+ break;
+ case BOOLEAN:
case LONG:
+ if (logicalType != null && !(logicalType instanceof LogicalTypes.TimestampMillis)) {
+ if (logicalType instanceof LogicalTypes.TimestampMicros) {
+ long dateIntValue = (long) fieldValue;
+ out = dateIntValue / 1000L;
+ } else {
+ LOGGER.warn("Actual type: INT, Logical Type: " + logicalType.getName());
+ out = fieldValue;
+ }
+ } else {
+ out = fieldValue;
+ }
+ break;
case DOUBLE:
case STRING:
out = fieldValue;
@@ -177,13 +210,27 @@ public class AvroCarbonWriter extends CarbonWriter {
String FieldName = avroField.name();
Schema childSchema = avroField.schema();
Schema.Type type = childSchema.getType();
+ LogicalType logicalType = childSchema.getLogicalType();
switch (type) {
case BOOLEAN:
return new Field(FieldName, DataTypes.BOOLEAN);
case INT:
- return new Field(FieldName, DataTypes.INT);
+ if (logicalType instanceof LogicalTypes.Date) {
+ return new Field(FieldName, DataTypes.DATE);
+ } else {
+ LOGGER.warn("Unsupported logical type. Considering Data Type as INT for " + childSchema
+ .getName());
+ return new Field(FieldName, DataTypes.INT);
+ }
case LONG:
- return new Field(FieldName, DataTypes.LONG);
+ if (logicalType instanceof LogicalTypes.TimestampMillis
+ || logicalType instanceof LogicalTypes.TimestampMicros) {
+ return new Field(FieldName, DataTypes.TIMESTAMP);
+ } else {
+ LOGGER.warn("Unsupported logical type. Considering Data Type as LONG for " + childSchema
+ .getName());
+ return new Field(FieldName, DataTypes.LONG);
+ }
case DOUBLE:
return new Field(FieldName, DataTypes.DOUBLE);
case STRING:
@@ -221,13 +268,27 @@ public class AvroCarbonWriter extends CarbonWriter {
private static StructField prepareSubFields(String FieldName, Schema childSchema) {
Schema.Type type = childSchema.getType();
+ LogicalType logicalType = childSchema.getLogicalType();
switch (type) {
case BOOLEAN:
return new StructField(FieldName, DataTypes.BOOLEAN);
case INT:
- return new StructField(FieldName, DataTypes.INT);
+ if (logicalType instanceof LogicalTypes.Date) {
+ return new StructField(FieldName, DataTypes.DATE);
+ } else {
+ LOGGER.warn("Unsupported logical type. Considering Data Type as INT for " + childSchema
+ .getName());
+ return new StructField(FieldName, DataTypes.INT);
+ }
case LONG:
- return new StructField(FieldName, DataTypes.LONG);
+ if (logicalType instanceof LogicalTypes.TimestampMillis
+ || logicalType instanceof LogicalTypes.TimestampMicros) {
+ return new StructField(FieldName, DataTypes.TIMESTAMP);
+ } else {
+ LOGGER.warn("Unsupported logical type. Considering Data Type as LONG for " + childSchema
+ .getName());
+ return new StructField(FieldName, DataTypes.LONG);
+ }
case DOUBLE:
return new StructField(FieldName, DataTypes.DOUBLE);
case STRING: