You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/10/09 15:50:25 UTC
[23/45] carbondata git commit: [CARBONDATA-2982] CarbonSchemaReader
support array
[CARBONDATA-2982] CarbonSchemaReader support array<string>
This PR fix the issue and change :
org.apache.carbondata.sdk.file.CarbonSchemaReader#readSchemaInDataFile
org.apache.carbondata.sdk.file.CarbonSchemaReader#readSchemaInIndexFile
This PR remove child schema
This closes #2780
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d8a51c9b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d8a51c9b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d8a51c9b
Branch: refs/heads/branch-1.5
Commit: d8a51c9bf314fb1cd5f6112e66eb04e776a0553d
Parents: 7d1fcb3
Author: xubo245 <xu...@huawei.com>
Authored: Fri Sep 28 11:47:22 2018 +0800
Committer: manishgupta88 <to...@gmail.com>
Committed: Wed Oct 3 16:24:38 2018 +0530
----------------------------------------------------------------------
.../examples/sdk/CarbonReaderExample.java | 4 +-
.../carbondata/examplesCI/RunExamples.scala | 5 ++
.../carbondata/sdk/file/CarbonSchemaReader.java | 14 +++-
.../carbondata/sdk/file/CarbonReaderTest.java | 86 ++++++++++++++++++++
4 files changed, 105 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8a51c9b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
index 9e80567..ef4ae7a 100644
--- a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
+++ b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
@@ -122,10 +122,11 @@ public class CarbonReaderExample {
for (int j = 0; j < arr.length; j++) {
System.out.print(arr[j] + " ");
}
+ assert (arr[0].equals("Hello"));
+ assert (arr[3].equals("Carbon"));
System.out.println();
i++;
}
- System.out.println("\nFinished");
reader.close();
// Read data
@@ -148,7 +149,6 @@ public class CarbonReaderExample {
row[5], row[6], row[7], row[8], row[9], row[10]));
i++;
}
- System.out.println("\nFinished");
reader2.close();
FileUtils.deleteDirectory(new File(path));
} catch (Throwable e) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8a51c9b/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala b/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala
index 2b9b999..6a13dc3 100644
--- a/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala
+++ b/examples/spark2/src/test/scala/org/apache/carbondata/examplesCI/RunExamples.scala
@@ -23,6 +23,7 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.examples._
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.examples.sdk.CarbonReaderExample
/**
* Test suite for examples
@@ -113,4 +114,8 @@ class RunExamples extends QueryTest with BeforeAndAfterAll {
test("ExternalTableExample") {
ExternalTableExample.exampleBody(spark)
}
+
+ test("CarbonReaderExample") {
+ CarbonReaderExample.main(null)
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8a51c9b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java
index d8882bc..e84a25a 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java
@@ -65,7 +65,15 @@ public class CarbonSchemaReader {
*/
public static Schema readSchemaInDataFile(String dataFilePath) throws IOException {
CarbonHeaderReader reader = new CarbonHeaderReader(dataFilePath);
- return new Schema(reader.readSchema());
+ List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
+ List<ColumnSchema> schemaList = reader.readSchema();
+ for (int i = 0; i < schemaList.size(); i++) {
+ ColumnSchema columnSchema = schemaList.get(i);
+ if (!(columnSchema.getColumnName().contains("."))) {
+ columnSchemaList.add(columnSchema);
+ }
+ }
+ return new Schema(columnSchemaList);
}
/**
@@ -97,7 +105,9 @@ public class CarbonSchemaReader {
List<org.apache.carbondata.format.ColumnSchema> table_columns =
readIndexHeader.getTable_columns();
for (org.apache.carbondata.format.ColumnSchema columnSchema : table_columns) {
- columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(columnSchema));
+ if (!(columnSchema.column_name.contains("."))) {
+ columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(columnSchema));
+ }
}
return new Schema(columnSchemaList);
} finally {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8a51c9b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index 8c5ffe5..8d95456 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -1435,5 +1435,91 @@ public class CarbonReaderTest extends TestCase {
FileUtils.deleteDirectory(new File("./testWriteFiles"));
}
+ @Test
+ public void testReadSchemaFromDataFileArrayString() {
+ String path = "./testWriteFiles";
+ try {
+ FileUtils.deleteDirectory(new File(path));
+ Field[] fields = new Field[11];
+ fields[0] = new Field("stringField", DataTypes.STRING);
+ fields[1] = new Field("shortField", DataTypes.SHORT);
+ fields[2] = new Field("intField", DataTypes.INT);
+ fields[3] = new Field("longField", DataTypes.LONG);
+ fields[4] = new Field("doubleField", DataTypes.DOUBLE);
+ fields[5] = new Field("boolField", DataTypes.BOOLEAN);
+ fields[6] = new Field("dateField", DataTypes.DATE);
+ fields[7] = new Field("timeField", DataTypes.TIMESTAMP);
+ fields[8] = new Field("decimalField", DataTypes.createDecimalType(8, 2));
+ fields[9] = new Field("varcharField", DataTypes.VARCHAR);
+ fields[10] = new Field("arrayField", DataTypes.createArrayType(DataTypes.STRING));
+ Map<String, String> map = new HashMap<>();
+ map.put("complex_delimiter_level_1", "#");
+ CarbonWriter writer = CarbonWriter.builder()
+ .outputPath(path)
+ .withLoadOptions(map)
+ .withCsvInput(new Schema(fields)).build();
+
+ for (int i = 0; i < 10; i++) {
+ String[] row2 = new String[]{
+ "robot" + (i % 10),
+ String.valueOf(i % 10000),
+ String.valueOf(i),
+ String.valueOf(Long.MAX_VALUE - i),
+ String.valueOf((double) i / 2),
+ String.valueOf(true),
+ "2019-03-02",
+ "2019-02-12 03:03:34",
+ "12.345",
+ "varchar",
+ "Hello#World#From#Carbon"
+ };
+ writer.write(row2);
+ }
+ writer.close();
+
+ File[] dataFiles = new File(path).listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name == null) {
+ return false;
+ }
+ return name.endsWith("carbondata");
+ }
+ });
+ if (dataFiles == null || dataFiles.length < 1) {
+ throw new RuntimeException("Carbon index file not exists.");
+ }
+ Schema schema = CarbonSchemaReader
+ .readSchemaInDataFile(dataFiles[0].getAbsolutePath())
+ .asOriginOrder();
+ // Transform the schema
+ String[] strings = new String[schema.getFields().length];
+ for (int i = 0; i < schema.getFields().length; i++) {
+ strings[i] = (schema.getFields())[i].getFieldName();
+ }
+
+ // Read data
+ CarbonReader reader = CarbonReader
+ .builder(path, "_temp")
+ .projection(strings)
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = (Object[]) reader.readNextRow();
+ assert (row[0].equals("robot" + i));
+ assert (row[2].equals(i));
+ assert (row[6].equals(17957));
+ Object[] arr = (Object[]) row[10];
+ assert (arr[0].equals("Hello"));
+ assert (arr[3].equals("Carbon"));
+ i++;
+ }
+ reader.close();
+ FileUtils.deleteDirectory(new File(path));
+ } catch (Throwable e) {
+ e.printStackTrace();
+ }
+ }
}