You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/03/25 22:06:57 UTC
[arrow] branch master updated: PARQUET-1477: [C++] sync thrift to
final crypto spec
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new a0b4a8c PARQUET-1477: [C++] sync thrift to final crypto spec
a0b4a8c is described below
commit a0b4a8ca5a6d80420d8c2f5783f1d21649d03fbf
Author: Gidon Gershinsky <gg...@gmail.com>
AuthorDate: Mon Mar 25 17:06:48 2019 -0500
PARQUET-1477: [C++] sync thrift to final crypto spec
Author: Gidon Gershinsky <gg...@gmail.com>
Closes #4022 from ggershinsky/p1477-sync-thrift-crypto-spec and squashes the following commits:
0ce831e05 <Gidon Gershinsky> sync thrift file to final crypto spec
---
cpp/src/parquet/parquet.thrift | 105 ++++++++++++++++++++++++++---------------
1 file changed, 66 insertions(+), 39 deletions(-)
diff --git a/cpp/src/parquet/parquet.thrift b/cpp/src/parquet/parquet.thrift
index efcc265..ff9907d 100644
--- a/cpp/src/parquet/parquet.thrift
+++ b/cpp/src/parquet/parquet.thrift
@@ -560,7 +560,7 @@ struct PageHeader {
/** Uncompressed page size in bytes (not including this header) **/
2: required i32 uncompressed_page_size
- /** Compressed page size in bytes (not including this header) **/
+ /** Compressed (and potentially encrypted) page size in bytes, not including this header **/
3: required i32 compressed_page_size
/** 32bit crc for the data below. This allows for disabling checksumming in HDFS
@@ -637,7 +637,8 @@ struct ColumnMetaData {
/** total byte size of all uncompressed pages in this column chunk (including the headers) **/
6: required i64 total_uncompressed_size
- /** total byte size of all compressed pages in this column chunk (including the headers) **/
+ /** total byte size of all compressed, and potentially encrypted, pages
+ * in this column chunk (including the headers) **/
7: required i64 total_compressed_size
/** Optional key/value metadata **/
@@ -667,9 +668,9 @@ struct EncryptionWithFooterKey {
struct EncryptionWithColumnKey {
/** Column path in schema **/
1: required list<string> path_in_schema
-
- /** Retrieval metadata of the column-specific key **/
- 2: optional binary column_key_metadata
+
+ /** Retrieval metadata of column encryption key **/
+ 2: optional binary key_metadata
}
union ColumnCryptoMetaData {
@@ -705,7 +706,10 @@ struct ColumnChunk {
7: optional i32 column_index_length
/** Crypto metadata of encrypted columns **/
- 8: optional ColumnCryptoMetaData crypto_meta_data
+ 8: optional ColumnCryptoMetaData crypto_metadata
+
+ /** Encrypted column metadata for this chunk **/
+ 9: optional binary encrypted_column_metadata
}
struct RowGroup {
@@ -729,8 +733,12 @@ struct RowGroup {
* in this row group **/
5: optional i64 file_offset
- /** Total byte size of all compressed column data in this row group **/
+ /** Total byte size of all compressed (and potentially encrypted) column data
+ * in this row group **/
6: optional i64 total_compressed_size
+
+ /** Row group ordinal in the file **/
+ 7: optional i16 ordinal
}
/** Empty struct to signal the order defined by the physical or logical type */
@@ -850,6 +858,35 @@ struct ColumnIndex {
5: optional list<i64> null_counts
}
+struct AesGcmV1 {
+ /** AAD prefix **/
+ 1: optional binary aad_prefix
+
+ /** Unique file identifier part of AAD suffix **/
+ 2: optional binary aad_file_unique
+
+ /** In files encrypted with AAD prefix without storing it,
+ * readers must supply the prefix **/
+ 3: optional bool supply_aad_prefix
+}
+
+struct AesGcmCtrV1 {
+ /** AAD prefix **/
+ 1: optional binary aad_prefix
+
+ /** Unique file identifier part of AAD suffix **/
+ 2: optional binary aad_file_unique
+
+ /** In files encrypted with AAD prefix without storing it,
+ * readers must supply the prefix **/
+ 3: optional bool supply_aad_prefix
+}
+
+union EncryptionAlgorithm {
+ 1: AesGcmV1 AES_GCM_V1
+ 2: AesGcmCtrV1 AES_GCM_CTR_V1
+}
+
/**
* Description for file metadata
*/
@@ -893,43 +930,33 @@ struct FileMetaData {
* regardless of column_orders.
*/
7: optional list<ColumnOrder> column_orders;
-}
-
-struct AesGcmV1 {
- /** Retrieval metadata of AAD used for encryption of pages and structures **/
- 1: optional binary aad_metadata
- /** If file IVs are comprised of a fixed part, and variable parts
- * (e.g. counter), keep the fixed part here **/
- 2: optional binary iv_prefix
-}
-
-struct AesGcmCtrV1 {
- /** Retrieval metadata of AAD used for encryption of structures **/
- 1: optional binary aad_metadata
-
- /** If file IVs are comprised of a fixed part, and variable parts
- * (e.g. counter), keep the fixed part here **/
- 2: optional binary gcm_iv_prefix
-
- 3: optional binary ctr_iv_prefix
-}
+ /**
+ * Encryption algorithm. This field is set only in encrypted files
+ * with plaintext footer. Files with encrypted footer store algorithm id
+ * in FileCryptoMetaData structure.
+ */
+ 8: optional EncryptionAlgorithm encryption_algorithm
-union EncryptionAlgorithm {
- 1: AesGcmV1 AES_GCM_V1
- 2: AesGcmCtrV1 AES_GCM_CTR_V1
+ /**
+ * Retrieval metadata of key used for signing the footer.
+ * Used only in encrypted files with plaintext footer.
+ */
+ 9: optional binary footer_signing_key_metadata
}
+/** Crypto metadata for files with encrypted footer **/
struct FileCryptoMetaData {
+ /**
+ * Encryption algorithm. This field is only used for files
+ * with encrypted footer. Files with plaintext footer store algorithm id
+ * inside footer (FileMetaData structure).
+ */
1: required EncryptionAlgorithm encryption_algorithm
-
- /** Parquet footer can be encrypted, or left as plaintext **/
- 2: required bool encrypted_footer
-
- /** Retrieval metadata of key used for encryption of footer,
+
+ /** Retrieval metadata of key used for encryption of footer,
* and (possibly) columns **/
- 3: optional binary footer_key_metadata
-
- /** Offset of Parquet footer (encrypted, or plaintext) **/
- 4: required i64 footer_offset
+ 2: optional binary key_metadata
}
+
+