You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/04/05 09:53:28 UTC
[03/12] incubator-carbondata git commit: [CARBONDATA-850] Fix the
comment definition issues of CarbonData thrift files
[CARBONDATA-850] Fix the comment definition issues of CarbonData thrift files
fix comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/970dfd99
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/970dfd99
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/970dfd99
Branch: refs/heads/branch-1.1
Commit: 970dfd99fbb36940c8feb0de07bed503ec859a41
Parents: 4d0d2d6
Author: chenliang613 <ch...@huawei.com>
Authored: Tue Apr 4 15:53:48 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Apr 4 18:06:50 2017 +0530
----------------------------------------------------------------------
format/src/main/thrift/carbondata.thrift | 190 ++++++++++---------
format/src/main/thrift/carbondata_index.thrift | 40 ++--
format/src/main/thrift/dictionary.thrift | 30 ++-
.../src/main/thrift/dictionary_metadata.thrift | 30 ++-
format/src/main/thrift/schema.thrift | 100 +++++-----
format/src/main/thrift/sort_index.thrift | 32 ++--
6 files changed, 208 insertions(+), 214 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/970dfd99/format/src/main/thrift/carbondata.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/carbondata.thrift b/format/src/main/thrift/carbondata.thrift
index 21ff29d..937108c 100644
--- a/format/src/main/thrift/carbondata.thrift
+++ b/format/src/main/thrift/carbondata.thrift
@@ -1,24 +1,22 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
/**
- * File format description for the carbon file format
+ * File format description for the CarbonData file format
*/
namespace java org.apache.carbondata.format
@@ -26,40 +24,40 @@ include "schema.thrift"
include "dictionary.thrift"
/**
-* Information about a segment, that represents one data load
-*/
+ * Information about a segment, that represents one data load
+ */
struct SegmentInfo{
- 1: required i32 num_cols; // Number of columns in this load, because schema can evolve . TODO: Check whether this is really required
+ 1: required i32 num_cols; // Number of columns in this load, because schema can evolve, different segments may have different columns
2: required list<i32> column_cardinalities; // Cardinality of columns
}
/**
-* Btree index of one blocklet
-*/
+ * Btree index of one blocklet
+ */
struct BlockletBTreeIndex{
1: required binary start_key; // Bit-packed start key of one blocklet
- 2: required binary end_key; // Bit-packed start key of one blocklet
+ 2: required binary end_key; // Bit-packed end key of one blocklet
}
/**
-* Min-max index of one blocklet
-*/
+ * Min-max index of one blocklet
+ */
struct BlockletMinMaxIndex{
1: required list<binary> min_values; //Min value of all columns of one blocklet Bit-Packed
2: required list<binary> max_values; //Max value of all columns of one blocklet Bit-Packed
}
/**
-* Index of one blocklet
-**/
+ * Index of one blocklet
+ */
struct BlockletIndex{
1: optional BlockletMinMaxIndex min_max_index;
2: optional BlockletBTreeIndex b_tree_index;
}
/**
-* Sort state of one column
-*/
+ * Sort state of one column
+ */
enum SortState{
SORT_NONE = 0; // Data is not sorted
SORT_NATIVE = 1; //Source data was sorted
@@ -67,157 +65,163 @@ enum SortState{
}
/**
-* Compressions supported by Carbon Data.
-*/
+ * Compressions supported by CarbonData.
+ */
enum CompressionCodec{
SNAPPY = 0;
}
/**
-* Represents the data of one dimension one dimension group in one blocklet
-*/
-// add a innger level placeholder for further I/O granulatity
+ * Represents the data of one column page or one column page group inside blocklet.
+ * Currently CarbonData using snappy for compression directly , user can specify the different compression by ChunkCompressionMeta.
+ * Currently not using it , keep it as for future extend ability.
+ */
struct ChunkCompressionMeta{
- 1: required CompressionCodec compression_codec; // the compressor used
- /** total byte size of all uncompressed pages in this column chunk (including the headers) **/
+ 1: required CompressionCodec compression_codec; // The compressor used
+ /** Total byte size of all uncompressed pages in this column chunk (including the headers) **/
2: required i64 total_uncompressed_size;
- /** total byte size of all compressed pages in this column chunk (including the headers) **/
+ /** Total byte size of all compressed pages in this column chunk (including the headers) **/
3: required i64 total_compressed_size;
}
/**
-* To handle space data with nulls
-*/
+ * To handle space data with nulls
+ */
struct PresenceMeta{
- 1: required bool represents_presence; // if true, ones in the bit stream reprents presence. otherwise represents absence
+ 1: required bool represents_presence; // If true, ones in the bit stream reprents presence. otherwise represents absence
2: required binary present_bit_stream; // Compressed bit stream representing the presence of null values
}
/**
-* Represents a chunk of data. The chunk can be a single column stored in Column Major format or a group of columns stored in Row Major Format.
-**/
+ * Represents a chunk of data. The chunk can be a single column stored in Column Major format or a group of columns stored in Row Major Format.
+ * For V1 format.
+ */
struct DataChunk{
- 1: required ChunkCompressionMeta chunk_meta; // the metadata of a chunk
- 2: required bool rowMajor; // whether this chunk is a row chunk or column chunk ? Decide whether this can be replace with counting od columnIDs
+ 1: required ChunkCompressionMeta chunk_meta; // The metadata of a chunk
+ 2: required bool rowMajor; // Whether this chunk is a row chunk or column chunk
/** The column IDs in this chunk, in the order in which the data is physically stored, will have atleast one column ID for columnar format, many column ID for row major format**/
3: required list<i32> column_ids;
4: required i64 data_page_offset; // Offset of data page
- 5: required i32 data_page_length; // length of data page
- 6: optional i64 rowid_page_offset; //offset of row id page, only if encoded using inverted index
- 7: optional i32 rowid_page_length; //length of row id page, only if encoded using inverted index
- 8: optional i64 rle_page_offset; // offset of rle page, only if RLE coded.
- 9: optional i32 rle_page_length; // length of rle page, only if RLE coded.
- 10: optional PresenceMeta presence; // information about presence of values in each row of this column chunk
+ 5: required i32 data_page_length; // Length of data page
+ 6: optional i64 rowid_page_offset; // Offset of row id page, only if encoded using inverted index
+ 7: optional i32 rowid_page_length; // Length of row id page, only if encoded using inverted index
+ 8: optional i64 rle_page_offset; // Offset of rle page, only if RLE coded.
+ 9: optional i32 rle_page_length; // Length of rle page, only if RLE coded.
+ 10: optional PresenceMeta presence; // Information about presence of values in each row of this column chunk
11: optional SortState sort_state;
12: optional list<schema.Encoding> encoders; // The List of encoders overriden at node level
- 13: optional list<binary> encoder_meta; // extra information required by encoders
+ 13: optional list<binary> encoder_meta; // Extra information required by encoders
}
/**
-* Represents a chunk of data. The chunk can be a single column stored in Column Major format or a group of columns stored in Row Major Format.
-**/
+ * Represents a chunk of data. The chunk can be a single column stored in Column Major format or a group of columns stored in Row Major Format.
+ * For V2 format.
+ */
struct DataChunk2{
- 1: required ChunkCompressionMeta chunk_meta; // the metadata of a chunk
- 2: required bool rowMajor; // whether this chunk is a row chunk or column chunk ? Decide whether this can be replace with counting od columnIDs
+ 1: required ChunkCompressionMeta chunk_meta; // The metadata of a chunk
+ 2: required bool rowMajor; // Whether this chunk is a row chunk or column chunk
/** The column IDs in this chunk, in the order in which the data is physically stored, will have atleast one column ID for columnar format, many column ID for row major format**/
- 3: required i32 data_page_length; // length of data page
- 4: optional i32 rowid_page_length; //length of row id page, only if encoded using inverted index
- 5: optional i32 rle_page_length; // length of rle page, only if RLE coded.
- 6: optional PresenceMeta presence; // information about presence of values in each row of this column chunk
+ 3: required i32 data_page_length; // Length of data page
+ 4: optional i32 rowid_page_length; // Length of row id page, only if encoded using inverted index
+ 5: optional i32 rle_page_length; // Length of rle page, only if RLE coded.
+ 6: optional PresenceMeta presence; // Information about presence of values in each row of this column chunk
7: optional SortState sort_state;
8: optional list<schema.Encoding> encoders; // The List of encoders overriden at node level
- 9: optional list<binary> encoder_meta; // extra information required by encoders
+ 9: optional list<binary> encoder_meta; // Extra information required by encoders
10: optional BlockletMinMaxIndex min_max;
11: optional i32 numberOfRowsInpage;
}
/**
-* Represents a chunk of data. The chunk can be a single column stored in Column Major format or a group of columns stored in Row Major Format.
-**/
+ * Represents a chunk of data. The chunk can be a single column stored in Column Major format or a group of columns stored in Row Major Format.
+ * For V3 format
+ */
struct DataChunk3{
- 1: required list<DataChunk2> data_chunk_list; // list of data chunk
- 2: optional list<i32> page_offset; // offset of each chunk
- 3: optional list<i32> page_length; // length of each chunk
+ 1: required list<DataChunk2> data_chunk_list; // List of data chunk
+ 2: optional list<i32> page_offset; // Offset of each chunk
+ 3: optional list<i32> page_length; // Length of each chunk
}
/**
-* Information about a blocklet
-*/
+ * Information about a blocklet for V1 format
+ */
struct BlockletInfo{
1: required i32 num_rows; // Number of rows in this blocklet
2: required list<DataChunk> column_data_chunks; // Information about all column chunks in this blocklet
}
/**
-* Information about a blocklet
-*/
+ * Information about a blocklet for V2 format
+ */
struct BlockletInfo2{
1: required i32 num_rows; // Number of rows in this blocklet
2: required list<i64> column_data_chunks_offsets; // Information about offsets all column chunks in this blocklet
3: required list<i16> column_data_chunks_length; // Information about length all column chunks in this blocklet
}
+
/**
-* Information about a blocklet
-*/
+ * Information about a blocklet for V3 format
+ */
struct BlockletInfo3{
1: required i32 num_rows; // Number of rows in this blocklet
2: required list<i64> column_data_chunks_offsets; // Information about offsets all column chunks in this blocklet
3: required list<i32> column_data_chunks_length; // Information about length all column chunks in this blocklet
4: required i64 dimension_offsets;
5: required i64 measure_offsets;
- 6: required i32 number_number_of_pages; // this is rquired for alter table, in case of alter table when filter is only selected on new added column this will help
+ 6: required i32 number_number_of_pages; // This is rquired for alter table, in case of alter table when filter is only selected on new added column this will help
}
+
/**
-* Footer for indexed carbon file
-*/
+ * Footer for indexed carbon file
+ */
struct FileFooter{
- 1: required i32 version; // version used for data compatibility
+ 1: required i32 version; // Version used for data compatibility
2: required i64 num_rows; // Total number of rows in this file
3: required list<schema.ColumnSchema> table_columns; // Description of columns in this file
4: required SegmentInfo segment_info; // Segment info (will be same/repeated for all files in this segment)
- 5: required list<BlockletIndex> blocklet_index_list; // blocklet index of all blocklets in this file
- 6: optional list<BlockletInfo> blocklet_info_list; // Information about blocklets of all columns in this file
- 7: optional list<BlockletInfo2> blocklet_info_list2; // Information about blocklets of all columns in this file
- 8: optional dictionary.ColumnDictionaryChunk dictionary; // blocklet local dictionary
+ 5: required list<BlockletIndex> blocklet_index_list; // Blocklet index of all blocklets in this file
+ 6: optional list<BlockletInfo> blocklet_info_list; // Information about blocklets of all columns in this file for V1 format
+ 7: optional list<BlockletInfo2> blocklet_info_list2; // Information about blocklets of all columns in this file for V2 format
+ 8: optional dictionary.ColumnDictionaryChunk dictionary; // Blocklet local dictionary
}
/**
-* Footer for indexed carbon file for V3 format
-*/
+ * Footer for indexed carbon file for V3 format
+ */
struct FileFooter3{
1: required i64 num_rows; // Total number of rows in this file
2: required SegmentInfo segment_info; // Segment info (will be same/repeated for all files in this segment)
- 3: required list<BlockletIndex> blocklet_index_list; // blocklet index of all blocklets in this file
- 4: optional list<BlockletInfo3> blocklet_info_list3; // Information about blocklets of all columns in this file
- 5: optional dictionary.ColumnDictionaryChunk dictionary; // blocklet local dictionary
+ 3: required list<BlockletIndex> blocklet_index_list; // Blocklet index of all blocklets in this file
+ 4: optional list<BlockletInfo3> blocklet_info_list3; // Information about blocklets of all columns in this file for V3 format
+ 5: optional dictionary.ColumnDictionaryChunk dictionary; // Blocklet local dictionary
}
/**
* Header for appendable carbon file
*/
struct FileHeader{
- 1: required i32 version; // version used for data compatibility
+ 1: required i32 version; // Version used for data compatibility
2: required list<schema.ColumnSchema> column_schema; // Description of columns in this file
- 3: optional bool is_footer_present; // to check whether footer is present or not
- 4: optional i64 time_stamp; // timestamp to compare column schema against master schema
+ 3: optional bool is_footer_present; // To check whether footer is present or not
+ 4: optional i64 time_stamp; // Timestamp to compare column schema against master schema
}
/**
* Mutation type
*/
enum MutationType {
- INSERT = 0; // used for inserting data
- DELETE = 1; // used for deleting data
+ INSERT = 0; // Used for inserting data
+ DELETE = 1; // Used for deleting data
}
/**
* Header for one blocklet in appendable carbon file
*/
struct BlockletHeader{
- 1: required i32 blocklet_length; // length of blocklet data
- 2: required MutationType mutation; // mutation type of this blocklet
- 3: required BlockletIndex blocklet_index; // index for the following blocklet
- 4: required BlockletInfo blocklet_info; // info for the following blocklet
- 5: optional dictionary.ColumnDictionaryChunk dictionary; // blocklet local dictionary
+ 1: required i32 blocklet_length; // Length of blocklet data
+ 2: required MutationType mutation; // Mutation type of this blocklet
+ 3: required BlockletIndex blocklet_index; // Index for the following blocklet
+ 4: required BlockletInfo blocklet_info; // Info for the following blocklet
+ 5: optional dictionary.ColumnDictionaryChunk dictionary; // Blocklet local dictionary
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/970dfd99/format/src/main/thrift/carbondata_index.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/carbondata_index.thrift b/format/src/main/thrift/carbondata_index.thrift
index ef6fb18..c055031 100644
--- a/format/src/main/thrift/carbondata_index.thrift
+++ b/format/src/main/thrift/carbondata_index.thrift
@@ -1,24 +1,22 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
/**
- * File format description for the carbon file format
+ * File format description for CarbonData index file for btree.
*/
namespace java org.apache.carbondata.format
@@ -26,21 +24,21 @@ include "schema.thrift"
include "carbondata.thrift"
/**
- * header information stored in index file
+ * Header information stored in index file
*/
struct IndexHeader{
- 1: required i32 version; // version used for data compatibility
+ 1: required i32 version; // Version used for data compatibility
2: required list<schema.ColumnSchema> table_columns; // Description of columns in this file
3: required carbondata.SegmentInfo segment_info; // Segment info (will be same/repeated for all files in this segment)
- 4: optional i32 bucket_id; //bucket number in which file contains
+ 4: optional i32 bucket_id; // Bucket number in which file contains
}
/**
- * block index information stored in index file for every block
+ * Block index information stored in index file for every block
*/
struct BlockIndex{
1: required i64 num_rows; // Total number of rows in this file
2: required string file_name; // Block file name
3: required i64 offset; // Offset of the footer
- 4: required carbondata.BlockletIndex block_index; // Block index
+ 4: required carbondata.BlockletIndex block_index; // Blocklet index
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/970dfd99/format/src/main/thrift/dictionary.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/dictionary.thrift b/format/src/main/thrift/dictionary.thrift
index 4eb0b25..2107b0e 100644
--- a/format/src/main/thrift/dictionary.thrift
+++ b/format/src/main/thrift/dictionary.thrift
@@ -1,24 +1,22 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
/**
- * File format description for carbon dictionary file
+ * File format description for CarbonData dictionary file
*/
namespace java org.apache.carbondata.format
struct ColumnDictionaryChunk {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/970dfd99/format/src/main/thrift/dictionary_metadata.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/dictionary_metadata.thrift b/format/src/main/thrift/dictionary_metadata.thrift
index d9ab197..f4cd849 100644
--- a/format/src/main/thrift/dictionary_metadata.thrift
+++ b/format/src/main/thrift/dictionary_metadata.thrift
@@ -1,24 +1,22 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
/**
- * File format description for carbon dictionary file
+ * File format description for CarbonData dictionary metadata file
*/
namespace java org.apache.carbondata.format
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/970dfd99/format/src/main/thrift/schema.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index a5f6e60..fba70ea 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -1,30 +1,28 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
/**
- * File format description for carbon schema file
+ * File format description for CarbonData schema file
*/
namespace java org.apache.carbondata.format
/**
-* The types supported by Carbon Data.
-*/
+ * The types supported by Carbon Data.
+ */
enum DataType {
STRING = 0,
SHORT = 1,
@@ -39,64 +37,64 @@ enum DataType {
}
/**
-* Encodings supported by Carbon Data. Not all encodings are valid for all types.
-* Certain Encodings can be chained.
-*/
+ * Encodings supported by Carbon Data. Not all encodings are valid for all types.
+ * Certain Encodings can be chained.
+ */
enum Encoding{
DICTIONARY = 0; // Identified that a column is dictionary encoded
DELTA = 1; // Identifies that a column delta encoded
RLE = 2; // Indetifies that a column is run length encoded
- INVERTED_INDEX = 3; // identifies that a column is encoded using inverted index, can be used only along with dictionary encoding
- BIT_PACKED = 4; // identifies that a column is encoded using bit packing, can be used only along with dictionary encoding
+ INVERTED_INDEX = 3; // Identifies that a column is encoded using inverted index, can be used only along with dictionary encoding
+ BIT_PACKED = 4; // Identifies that a column is encoded using bit packing, can be used only along with dictionary encoding
DIRECT_DICTIONARY = 5; // Identifies that a column is direct dictionary encoded
}
/**
-* Description of a Column for both dimension and measure
-*/
+ * Description of a Column for both dimension and measure
+ */
//TODO:where to put the CSV column name and carbon table column name mapping? should not keep in schema
struct ColumnSchema{
1: required DataType data_type;
/**
- * Name of the column. If it is a complex data type, we follow a naming rule grand_parent_column.parent_column.child_column
- * For Array types, two columns will be stored one for the array type and one for the primitive type with the name parent_column.value
- */
+ * Name of the column. If it is a complex data type, we follow a naming rule grand_parent_column.parent_column.child_column
+ * For Array types, two columns will be stored one for the array type and one for the primitive type with the name parent_column.value
+ */
2: required string column_name; //
3: required string column_id; // Unique ID for a column. if this is dimension, it is an unique ID that used in dictionary
- 4: required bool columnar; // wether it is stored as columnar format or row format
+ 4: required bool columnar; // Whether it is stored as columnar format or row format
5: required list<Encoding> encoders; // List of encoders that are chained to encode the data for this column
6: required bool dimension; // Whether the column is a dimension or measure
7: optional i32 column_group_id; // The group ID for column used for row format columns, where in columns in each group are chunked together.
/**
- * Used when this column contains mantissa data.
- */
+ * Used when this column contains mantissa data.
+ */
8: optional i32 scale;
9: optional i32 precision;
/** Nested fields. Since thrift does not support nested fields,
- * the nesting is flattened to a single list by a depth-first traversal.
- * The children count is used to construct the nested relationship.
- * This field is not set when the element is a primitive type
- */
+ * the nesting is flattened to a single list by a depth-first traversal.
+ * The children count is used to construct the nested relationship.
+ * This field is not set when the element is a primitive type
+ */
10: optional i32 num_child;
/**
- * Used when this column is part of an aggregate table.
- */
+ * Used when this column is part of an aggregate table.
+ */
11: optional string aggregate_function;
12: optional binary default_value;
13: optional map<string,string> columnProperties;
- /**
- * To specify the visibily of the column by default its false
- */
+ /**
+ * To specify the visibily of the column by default its false
+ */
14: optional bool invisible;
/**
- * column reference id
+ * Column reference id
*/
15: optional string columnReferenceId;
/**
@@ -106,8 +104,8 @@ struct ColumnSchema{
}
/**
-* Description of One Schema Change, contains list of added columns and deleted columns
-*/
+ * Description of One Schema Change, contains list of added columns and deleted columns
+ */
struct SchemaEvolutionEntry{
1: required i64 time_stamp;
2: optional list<ColumnSchema> added;
@@ -116,29 +114,29 @@ struct SchemaEvolutionEntry{
}
/**
-* History of schema evolution
-*/
+ * History of schema evolution
+ */
struct SchemaEvolution{
1: required list<SchemaEvolutionEntry> schema_evolution_history;
}
/**
-* Bucketing information of fields on table
-**/
+ * Bucketing information of fields on table
+ */
struct BucketingInfo{
1: required list<ColumnSchema> table_columns;
2: required i32 number_of_buckets;
}
/**
-* The description of table schema
-*/
+ * The description of table schema
+ */
struct TableSchema{
1: required string table_id; // ID used to
2: required list<ColumnSchema> table_columns; // Columns in the table
3: required SchemaEvolution schema_evolution; // History of schema evolution of this table
- 4: optional map<string,string> tableProperties; // table properties configured bu the user
- 5: optional BucketingInfo bucketingInfo; // bucketing information
+ 4: optional map<string,string> tableProperties; // Table properties configured by the user
+ 5: optional BucketingInfo bucketingInfo; // Bucketing information
}
struct TableInfo{
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/970dfd99/format/src/main/thrift/sort_index.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/sort_index.thrift b/format/src/main/thrift/sort_index.thrift
index 45686dd..4c58318 100644
--- a/format/src/main/thrift/sort_index.thrift
+++ b/format/src/main/thrift/sort_index.thrift
@@ -1,28 +1,26 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
/**
- * File format description for carbon sort index file
+ * File format description for CarbonData sort index file
*/
namespace java org.apache.carbondata.format
struct ColumnSortInfo {
- 1: list<i32> sort_index; // the surrogate values sorted by the original value order
+ 1: list<i32> sort_index; // The surrogate values sorted by the original value order
2: list<i32> sort_index_inverted; // The inverted sort index
}