You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/06/05 10:42:00 UTC
[15/26] carbondata git commit: [CARBONDATA-2558] Optimize carbon
schema reader interface of SDK
[CARBONDATA-2558] Optimize carbon schema reader interface of SDK
Optimize carbon schema reader interface of SDK
1.create CarbonSchemaReader and move schema read interface from CarbonReader to CarbonSchemaReader
2.change the return type from List to SDK Schema, remove the tableInfo return type
3.Optimize the document
This closes #2353
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6d245b9d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6d245b9d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6d245b9d
Branch: refs/heads/branch-1.4
Commit: 6d245b9daf0dce4d3366c540c28bf27a4e38ac90
Parents: b792b3e
Author: xubo245 <xu...@huawei.com>
Authored: Tue May 29 17:07:10 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Jun 5 16:04:20 2018 +0530
----------------------------------------------------------------------
docs/sdk-guide.md | 107 +++++++++++++++---
.../carbondata/sdk/file/CarbonReader.java | 85 ---------------
.../carbondata/sdk/file/CarbonSchemaReader.java | 108 +++++++++++++++++++
.../org/apache/carbondata/sdk/file/Field.java | 16 +++
.../org/apache/carbondata/sdk/file/Schema.java | 31 ++++++
.../carbondata/store/MetaCachedCarbonStore.java | 13 ++-
.../carbondata/sdk/file/CarbonReaderTest.java | 88 ++++++++-------
7 files changed, 305 insertions(+), 143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6d245b9d/docs/sdk-guide.md
----------------------------------------------------------------------
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index 328a845..3c575fe 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -437,20 +437,6 @@ Find example code at [CarbonReaderExample](https://github.com/apache/carbondata/
```
/**
- * Read carbondata file and return the schema
- */
- public static List<ColumnSchema> readSchemaInDataFile(String dataFilePath);
-```
-
-```
- /**
- * Read schema file and return table info object
- */
- public static TableInfo readSchemaFile(String schemaFilePath);
-```
-
-```
- /**
* Return true if has next row
*/
public boolean hasNext();
@@ -598,4 +584,97 @@ Find example code at [CarbonReaderExample](https://github.com/apache/carbondata/
*/
public <T> CarbonReader<T> build();
```
+### Class org.apache.carbondata.sdk.file.CarbonSchemaReader
+```
+ /**
+ * Read schema file and return the schema
+ *
+ * @param schemaFilePath complete path including schema file name
+ * @return schema object
+ * @throws IOException
+ */
+ public static Schema readSchemaInSchemaFile(String schemaFilePath);
+```
+
+```
+ /**
+ * Read carbondata file and return the schema
+ *
+ * @param dataFilePath complete path including carbondata file name
+ * @return Schema object
+ * @throws IOException
+ */
+ public static Schema readSchemaInDataFile(String dataFilePath);
+```
+
+```
+ /**
+ * Read carbonindex file and return the schema
+ *
+ * @param indexFilePath complete path including index file name
+ * @return schema object
+ * @throws IOException
+ */
+ public static Schema readSchemaInIndexFile(String indexFilePath);
+```
+
+### Class org.apache.carbondata.sdk.file.Schema
+```
+ /**
+ * construct a schema with fields
+ * @param fields
+ */
+ public Schema(Field[] fields);
+```
+
+```
+ /**
+ * construct a schema with List<ColumnSchema>
+ *
+ * @param columnSchemaList column schema list
+ */
+ public Schema(List<ColumnSchema> columnSchemaList);
+```
+
+```
+ /**
+ * Create a Schema using JSON string, for example:
+ * [
+ * {"name":"string"},
+ * {"age":"int"}
+ * ]
+ * @param json specified as string
+ * @return Schema
+ */
+ public static Schema parseJson(String json);
+```
+
+```
+ /**
+ * Sort the schema order as original order
+ *
+ * @return Schema object
+ */
+ public Schema asOriginOrder();
+```
+
+### Class org.apache.carbondata.sdk.file.Field
+```
+ /**
+ * Field Constructor
+ * @param name name of the field
+ * @param type datatype of field, specified in strings.
+ */
+ public Field(String name, String type);
+```
+
+```
+ /**
+ * Construct Field from ColumnSchema
+ *
+ * @param columnSchema ColumnSchema, Store the information about the column meta data
+ */
+ public Field(ColumnSchema columnSchema);
+```
+
Find S3 example code at [SDKS3Example](https://github.com/apache/carbondata/blob/master/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java) in the CarbonData repo.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6d245b9d/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
index 9ae940b..60ead05 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
@@ -17,26 +17,11 @@
package org.apache.carbondata.sdk.file;
-import java.io.DataInputStream;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
import java.util.List;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.converter.SchemaConverter;
-import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
-import org.apache.carbondata.core.metadata.schema.table.TableInfo;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.reader.CarbonHeaderReader;
-import org.apache.carbondata.core.reader.CarbonIndexFileReader;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-import static org.apache.carbondata.core.util.CarbonUtil.thriftColumnSchemaToWrapperColumnSchema;
import org.apache.hadoop.mapreduce.RecordReader;
@@ -104,76 +89,6 @@ public class CarbonReader<T> {
}
/**
- * Read carbondata file and return the schema
- */
- public static List<ColumnSchema> readSchemaInDataFile(String dataFilePath) throws IOException {
- CarbonHeaderReader reader = new CarbonHeaderReader(dataFilePath);
- return reader.readSchema();
- }
-
- /**
- * Read carbonindex file and return the schema
- *
- * @param indexFilePath complete path including index file name
- * @return null, if the index file is not present in the path.
- * List<ColumnSchema> from the index file.
- * @throws IOException
- */
- public static List<ColumnSchema> readSchemaInIndexFile(String indexFilePath) throws IOException {
- CarbonFile indexFile =
- FileFactory.getCarbonFile(indexFilePath, FileFactory.getFileType(indexFilePath));
- if (!indexFile.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
- throw new IOException("Not an index file name");
- }
- // read schema from the first index file
- DataInputStream dataInputStream =
- FileFactory.getDataInputStream(indexFilePath, FileFactory.getFileType(indexFilePath));
- byte[] bytes = new byte[(int) indexFile.getSize()];
- try {
- //get the file in byte buffer
- dataInputStream.readFully(bytes);
- CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
- // read from byte buffer.
- indexReader.openThriftReader(bytes);
- // get the index header
- org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();
- List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
- List<org.apache.carbondata.format.ColumnSchema> table_columns =
- readIndexHeader.getTable_columns();
- for (org.apache.carbondata.format.ColumnSchema columnSchema : table_columns) {
- columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(columnSchema));
- }
- return columnSchemaList;
- } finally {
- dataInputStream.close();
- }
- }
-
- /**
- * Read CarbonData file and return the user schema,
- * the schema order is the same as user save schema
- */
- public static List<ColumnSchema> readUserSchema(String indexFilePath) throws IOException {
- List<ColumnSchema> columnSchemas = readSchemaInIndexFile(indexFilePath);
- Collections.sort(columnSchemas, new Comparator<ColumnSchema>() {
- @Override
- public int compare(ColumnSchema o1, ColumnSchema o2) {
- return Integer.compare(o1.getSchemaOrdinal(), o2.getSchemaOrdinal());
- }
- });
- return columnSchemas;
- }
-
- /**
- * Read schema file and return table info object
- */
- public static TableInfo readSchemaFile(String schemaFilePath) throws IOException {
- org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil.readSchemaFile(schemaFilePath);
- SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
- return schemaConverter.fromExternalToWrapperTableInfo(tableInfo, "", "", "");
- }
-
- /**
* Close reader
*
* @throws IOException
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6d245b9d/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java
new file mode 100644
index 0000000..d8882bc
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.file;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.converter.SchemaConverter;
+import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.reader.CarbonHeaderReader;
+import org.apache.carbondata.core.reader.CarbonIndexFileReader;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import static org.apache.carbondata.core.util.CarbonUtil.thriftColumnSchemaToWrapperColumnSchema;
+
+/**
+ * Schema reader for carbon files, including carbondata file, carbonindex file, and schema file
+ */
+public class CarbonSchemaReader {
+
+ /**
+ * Read schema file and return the schema
+ *
+ * @param schemaFilePath complete path including schema file name
+ * @return schema object
+ * @throws IOException
+ */
+ public static Schema readSchemaInSchemaFile(String schemaFilePath) throws IOException {
+ org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil.readSchemaFile(schemaFilePath);
+ SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
+ List<ColumnSchema> schemaList = schemaConverter
+ .fromExternalToWrapperTableInfo(tableInfo, "", "", "")
+ .getFactTable()
+ .getListOfColumns();
+ return new Schema(schemaList);
+ }
+
+ /**
+ * Read carbondata file and return the schema
+ *
+ * @param dataFilePath complete path including carbondata file name
+ * @return Schema object
+ * @throws IOException
+ */
+ public static Schema readSchemaInDataFile(String dataFilePath) throws IOException {
+ CarbonHeaderReader reader = new CarbonHeaderReader(dataFilePath);
+ return new Schema(reader.readSchema());
+ }
+
+ /**
+ * Read carbonindex file and return the schema
+ *
+ * @param indexFilePath complete path including index file name
+ * @return schema object
+ * @throws IOException
+ */
+ public static Schema readSchemaInIndexFile(String indexFilePath) throws IOException {
+ CarbonFile indexFile =
+ FileFactory.getCarbonFile(indexFilePath, FileFactory.getFileType(indexFilePath));
+ if (!indexFile.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
+ throw new IOException("Not an index file name");
+ }
+ // read schema from the first index file
+ DataInputStream dataInputStream =
+ FileFactory.getDataInputStream(indexFilePath, FileFactory.getFileType(indexFilePath));
+ byte[] bytes = new byte[(int) indexFile.getSize()];
+ try {
+ //get the file in byte buffer
+ dataInputStream.readFully(bytes);
+ CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
+ // read from byte buffer.
+ indexReader.openThriftReader(bytes);
+ // get the index header
+ org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();
+ List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
+ List<org.apache.carbondata.format.ColumnSchema> table_columns =
+ readIndexHeader.getTable_columns();
+ for (org.apache.carbondata.format.ColumnSchema columnSchema : table_columns) {
+ columnSchemaList.add(thriftColumnSchemaToWrapperColumnSchema(columnSchema));
+ }
+ return new Schema(columnSchemaList);
+ } finally {
+ dataInputStream.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6d245b9d/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
index 0db3bc5..6d4cfd9 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.sdk.file;
+import java.util.LinkedList;
import java.util.List;
import org.apache.carbondata.common.annotations.InterfaceAudience;
@@ -24,6 +25,7 @@ import org.apache.carbondata.common.annotations.InterfaceStability;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.datatype.StructField;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
/**
* A field represent one column
@@ -126,6 +128,20 @@ public class Field {
this.type = type;
}
+ /**
+ * Construct Field from ColumnSchema
+ *
+ * @param columnSchema ColumnSchema, Store the information about the column meta data
+ */
+ public Field(ColumnSchema columnSchema) {
+ this.name = columnSchema.getColumnName();
+ this.type = columnSchema.getDataType();
+ children = new LinkedList<>();
+ schemaOrdinal = columnSchema.getSchemaOrdinal();
+ precision = columnSchema.getPrecision();
+ scale = columnSchema.getScale();
+ }
+
public String getFieldName() {
return name;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6d245b9d/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java
index 31c202d..6131d45 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java
@@ -18,9 +18,13 @@
package org.apache.carbondata.sdk.file;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import com.google.gson.GsonBuilder;
import com.google.gson.TypeAdapter;
@@ -45,6 +49,18 @@ public class Schema {
}
/**
+ * construct a schema with List<ColumnSchema>
+ *
+ * @param columnSchemaList column schema list
+ */
+ public Schema(List<ColumnSchema> columnSchemaList) {
+ fields = new Field[columnSchemaList.size()];
+ for (int i = 0; i < columnSchemaList.size(); i++) {
+ fields[i] = new Field(columnSchemaList.get(i));
+ }
+ }
+
+ /**
* Create a Schema using JSON string, for example:
* [
* {"name":"string"},
@@ -77,4 +93,19 @@ public class Schema {
public Field[] getFields() {
return fields;
}
+
+ /**
+ * Sort the schema order as original order
+ *
+ * @return Schema object
+ */
+ public Schema asOriginOrder() {
+ Arrays.sort(fields, new Comparator<Field>() {
+ @Override
+ public int compare(Field o1, Field o2) {
+ return Integer.compare(o1.getSchemaOrdinal(), o2.getSchemaOrdinal());
+ }
+ });
+ return this;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6d245b9d/store/sdk/src/main/java/org/apache/carbondata/store/MetaCachedCarbonStore.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/store/MetaCachedCarbonStore.java b/store/sdk/src/main/java/org/apache/carbondata/store/MetaCachedCarbonStore.java
index d847e67..e43f750 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/store/MetaCachedCarbonStore.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/store/MetaCachedCarbonStore.java
@@ -22,10 +22,12 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.metadata.converter.SchemaConverter;
+import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
-import org.apache.carbondata.sdk.file.CarbonReader;
/**
* A CarbonStore base class that caches CarbonTable object
@@ -40,9 +42,12 @@ abstract class MetaCachedCarbonStore implements CarbonStore {
if (cache.containsKey(path)) {
return cache.get(path);
}
- TableInfo schema = CarbonReader.readSchemaFile(CarbonTablePath.getSchemaFilePath(path));
- schema.setTablePath(path);
- CarbonTable table = CarbonTable.buildFromTableInfo(schema);
+ org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil
+ .readSchemaFile(CarbonTablePath.getSchemaFilePath(path));
+ SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
+ TableInfo tableInfo1 = schemaConverter.fromExternalToWrapperTableInfo(tableInfo, "", "", "");
+ tableInfo1.setTablePath(path);
+ CarbonTable table = CarbonTable.buildFromTableInfo(tableInfo1);
cache.put(path, table);
return table;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6d245b9d/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index 30d4091..7a2a765 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -20,9 +20,8 @@ package org.apache.carbondata.sdk.file;
import java.io.*;
import java.sql.Date;
import java.sql.Timestamp;
-import java.util.Collections;
+import java.util.Arrays;
import java.util.Comparator;
-import java.util.List;
import org.apache.avro.generic.GenericData;
import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
@@ -30,8 +29,6 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.schema.table.TableInfo;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -200,12 +197,12 @@ public class CarbonReaderTest extends TestCase {
});
Assert.assertTrue(dataFiles != null);
Assert.assertTrue(dataFiles.length > 0);
- List<ColumnSchema> columns = CarbonReader.readSchemaInDataFile(dataFiles[0].getAbsolutePath());
- Assert.assertTrue(columns.size() == 2);
- Assert.assertEquals("name", columns.get(0).getColumnName());
- Assert.assertEquals("age", columns.get(1).getColumnName());
- Assert.assertEquals(DataTypes.STRING, columns.get(0).getDataType());
- Assert.assertEquals(DataTypes.INT, columns.get(1).getDataType());
+ Schema schema = CarbonSchemaReader.readSchemaInDataFile(dataFiles[0].getAbsolutePath());
+ Assert.assertTrue(schema.getFields().length == 2);
+ Assert.assertEquals("name", (schema.getFields())[0].getFieldName());
+ Assert.assertEquals("age", (schema.getFields())[1].getFieldName());
+ Assert.assertEquals(DataTypes.STRING, (schema.getFields())[0].getDataType());
+ Assert.assertEquals(DataTypes.INT, (schema.getFields())[1].getDataType());
FileUtils.deleteDirectory(new File(path));
}
@@ -228,20 +225,33 @@ public class CarbonReaderTest extends TestCase {
});
Assert.assertTrue(dataFiles != null);
Assert.assertTrue(dataFiles.length > 0);
- TableInfo tableInfo = CarbonReader.readSchemaFile(dataFiles[0].getAbsolutePath());
- Assert.assertEquals(2, tableInfo.getFactTable().getListOfColumns().size());
- List<ColumnSchema> columns = tableInfo.getFactTable().getListOfColumns();
- Assert.assertEquals(2, columns.size());
- Assert.assertEquals("name", columns.get(0).getColumnName());
- Assert.assertEquals("age", columns.get(1).getColumnName());
- Assert.assertEquals(DataTypes.STRING, columns.get(0).getDataType());
- Assert.assertEquals(DataTypes.INT, columns.get(1).getDataType());
+ Schema schema = CarbonSchemaReader.readSchemaInSchemaFile(dataFiles[0].getAbsolutePath());
+
+ // sort the schema
+ Arrays.sort(schema.getFields(), new Comparator<Field>() {
+ @Override
+ public int compare(Field o1, Field o2) {
+ return Integer.compare(o1.getSchemaOrdinal(), o2.getSchemaOrdinal());
+ }
+ });
+
+ // Transform the schema
+ String[] strings = new String[schema.getFields().length];
+ for (int i = 0; i < schema.getFields().length; i++) {
+ strings[i] = (schema.getFields())[i].getFieldName();
+ }
+
+ Assert.assertEquals(2, schema.getFields().length);
+
+ Assert.assertEquals("name", (schema.getFields())[0].getFieldName());
+ Assert.assertEquals("age", (schema.getFields())[1].getFieldName());
+ Assert.assertEquals(DataTypes.STRING, (schema.getFields())[0].getDataType());
+ Assert.assertEquals(DataTypes.INT, (schema.getFields())[1].getDataType());
FileUtils.deleteDirectory(new File(path));
}
-
@Test
public void testWriteAndReadFilesNonTransactional() throws IOException, InterruptedException {
String path = "./testWriteFiles";
@@ -473,22 +483,20 @@ public class CarbonReaderTest extends TestCase {
return name.endsWith("schema");
}
});
- TableInfo tableInfo = CarbonReader.readSchemaFile(dataFiles[0].getAbsolutePath());
-
- List<ColumnSchema> columns = tableInfo.getFactTable().getListOfColumns();
+ Schema schema = CarbonSchemaReader.readSchemaInSchemaFile(dataFiles[0].getAbsolutePath());
// sort the schema
- Collections.sort(tableInfo.getFactTable().getListOfColumns(), new Comparator<ColumnSchema>() {
+ Arrays.sort(schema.getFields(), new Comparator<Field>() {
@Override
- public int compare(ColumnSchema o1, ColumnSchema o2) {
+ public int compare(Field o1, Field o2) {
return Integer.compare(o1.getSchemaOrdinal(), o2.getSchemaOrdinal());
}
});
// Transform the schema
- String[] strings= new String[columns.size()];
- for (int i = 0; i < columns.size(); i++) {
- strings[i]= columns.get(i).getColumnName();
+ String[] strings = new String[schema.getFields().length];
+ for (int i = 0; i < schema.getFields().length; i++) {
+ strings[i] = (schema.getFields())[i].getFieldName();
}
File segmentFolder = new File(CarbonTablePath.getSegmentPath(path, "null"));
@@ -591,20 +599,20 @@ public class CarbonReaderTest extends TestCase {
}
});
- List<ColumnSchema> columns = CarbonReader.readSchemaInDataFile(dataFiles2[0].getAbsolutePath());
+ Schema schema = CarbonSchemaReader.readSchemaInDataFile(dataFiles2[0].getAbsolutePath());
// sort the schema
- Collections.sort(columns, new Comparator<ColumnSchema>() {
+ Arrays.sort(schema.getFields(), new Comparator<Field>() {
@Override
- public int compare(ColumnSchema o1, ColumnSchema o2) {
+ public int compare(Field o1, Field o2) {
return Integer.compare(o1.getSchemaOrdinal(), o2.getSchemaOrdinal());
}
});
// Transform the schema
- String[] strings= new String[columns.size()];
- for (int i = 0; i < columns.size(); i++) {
- strings[i]= columns.get(i).getColumnName();
+ String[] strings = new String[schema.getFields().length];
+ for (int i = 0; i < schema.getFields().length; i++) {
+ strings[i] = (schema.getFields())[i].getFieldName();
}
File segmentFolder = new File(CarbonTablePath.getSegmentPath(path, "null"));
@@ -704,12 +712,12 @@ public class CarbonReaderTest extends TestCase {
}
});
- List<ColumnSchema> columns = CarbonReader.readUserSchema(dataFiles2[0].getAbsolutePath());
+ Schema schema = CarbonSchemaReader.readSchemaInIndexFile(dataFiles2[0].getAbsolutePath()).asOriginOrder();
// Transform the schema
- String[] strings= new String[columns.size()];
- for (int i = 0; i < columns.size(); i++) {
- strings[i]= columns.get(i).getColumnName();
+ String[] strings = new String[schema.getFields().length];
+ for (int i = 0; i < schema.getFields().length; i++) {
+ strings[i] = (schema.getFields())[i].getFieldName();
}
File segmentFolder = new File(CarbonTablePath.getSegmentPath(path, "null"));
@@ -936,10 +944,10 @@ public class CarbonReaderTest extends TestCase {
}
});
- List<ColumnSchema> columns = CarbonReader.readUserSchema(dataFiles2[0].getAbsolutePath());
+ Schema schema = CarbonSchemaReader.readSchemaInIndexFile(dataFiles2[0].getAbsolutePath()).asOriginOrder();
- for (int i = 0; i < columns.size(); i++) {
- System.out.println(columns.get(i).getColumnName() + "\t" + columns.get(i).getSchemaOrdinal());
+ for (int i = 0; i < schema.getFields().length; i++) {
+ System.out.println((schema.getFields())[i].getFieldName() + "\t" + schema.getFields()[i].getSchemaOrdinal());
}
FileUtils.deleteDirectory(new File(path));
}