You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2023/12/26 11:37:11 UTC
(camel) branch avro created (now dee63f7f3e3)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a change to branch avro
in repository https://gitbox.apache.org/repos/asf/camel.git
at dee63f7f3e3 CAMEL-19143: parquet avro dataformat - Add enum for compression levels.
This branch includes the following new commits:
new dee63f7f3e3 CAMEL-19143: parquet avro dataformat - Add enum for compression levels.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(camel) 01/01: CAMEL-19143: parquet avro dataformat - Add enum for compression levels.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch avro
in repository https://gitbox.apache.org/repos/asf/camel.git
commit dee63f7f3e3445c2dc24c9671f8b3b30e47d300c
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Tue Dec 26 12:37:00 2023 +0100
CAMEL-19143: parquet avro dataformat - Add enum for compression levels.
---
.../resources/org/apache/camel/catalog/dataformats/parquetAvro.json | 2 +-
.../resources/org/apache/camel/catalog/models/parquetAvro.json | 2 +-
.../org/apache/camel/dataformat/parquet/avro/parquetAvro.json | 2 +-
.../apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java | 4 +---
.../resources/org/apache/camel/model/dataformat/parquetAvro.json | 2 +-
.../org/apache/camel/model/dataformat/ParquetAvroDataFormat.java | 5 +----
6 files changed, 6 insertions(+), 11 deletions(-)
diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dataformats/parquetAvro.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dataformats/parquetAvro.json
index 45cc3d1217f..70a3b613d67 100644
--- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dataformats/parquetAvro.json
+++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dataformats/parquetAvro.json
@@ -17,7 +17,7 @@
},
"properties": {
"id": { "index": 0, "kind": "attribute", "displayName": "Id", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The id of this node" },
- "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
+ "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "enum", "javaType": "java.lang.String", "enum": [ "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "ZSTD", "LZ4_RAW" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
"unmarshalType": { "index": 2, "kind": "attribute", "displayName": "Unmarshal Type", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Class to use when (un)marshalling. If omitted, parquet files are converted into Avro's GenericRecords for unmarshalling and input objects are assumed as GenericRecords for marshalling." },
"lazyLoad": { "index": 3, "kind": "attribute", "displayName": "Lazy Load", "required": false, "type": "boolean", "javaType": "java.lang.Boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the unmarshalling should produce an iterator of records or read all the records at once." }
}
diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/parquetAvro.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/parquetAvro.json
index 048b6aa31e3..061701c4db6 100644
--- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/parquetAvro.json
+++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/parquetAvro.json
@@ -14,7 +14,7 @@
},
"properties": {
"id": { "index": 0, "kind": "attribute", "displayName": "Id", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The id of this node" },
- "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
+ "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "enum", "javaType": "java.lang.String", "enum": [ "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "ZSTD", "LZ4_RAW" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
"unmarshalType": { "index": 2, "kind": "attribute", "displayName": "Unmarshal Type", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Class to use when (un)marshalling. If omitted, parquet files are converted into Avro's GenericRecords for unmarshalling and input objects are assumed as GenericRecords for marshalling." },
"lazyLoad": { "index": 3, "kind": "attribute", "displayName": "Lazy Load", "required": false, "type": "boolean", "javaType": "java.lang.Boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the unmarshalling should produce an iterator of records or read all the records at once." }
}
diff --git a/components/camel-parquet-avro/src/generated/resources/org/apache/camel/dataformat/parquet/avro/parquetAvro.json b/components/camel-parquet-avro/src/generated/resources/org/apache/camel/dataformat/parquet/avro/parquetAvro.json
index 45cc3d1217f..70a3b613d67 100644
--- a/components/camel-parquet-avro/src/generated/resources/org/apache/camel/dataformat/parquet/avro/parquetAvro.json
+++ b/components/camel-parquet-avro/src/generated/resources/org/apache/camel/dataformat/parquet/avro/parquetAvro.json
@@ -17,7 +17,7 @@
},
"properties": {
"id": { "index": 0, "kind": "attribute", "displayName": "Id", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The id of this node" },
- "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
+ "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "enum", "javaType": "java.lang.String", "enum": [ "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "ZSTD", "LZ4_RAW" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
"unmarshalType": { "index": 2, "kind": "attribute", "displayName": "Unmarshal Type", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Class to use when (un)marshalling. If omitted, parquet files are converted into Avro's GenericRecords for unmarshalling and input objects are assumed as GenericRecords for marshalling." },
"lazyLoad": { "index": 3, "kind": "attribute", "displayName": "Lazy Load", "required": false, "type": "boolean", "javaType": "java.lang.Boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the unmarshalling should produce an iterator of records or read all the records at once." }
}
diff --git a/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java b/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java
index a4809da2bc8..5844f39231d 100644
--- a/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java
+++ b/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java
@@ -55,9 +55,7 @@ public class ParquetAvroDataFormat extends ServiceSupport implements DataFormat,
private static final DefaultUuidGenerator DEFAULT_UUID_GENERATOR = new DefaultUuidGenerator();
private CompressionCodecName compressionCodecName = GZIP;
-
private Class<?> unmarshalType;
-
private boolean lazyLoad;
@Override
@@ -86,7 +84,7 @@ public class ParquetAvroDataFormat extends ServiceSupport implements DataFormat,
schema = ReflectData.AllowNull.get().getSchema(unmarshalType); // generate nullable fields
model = ReflectData.get();
} catch (AvroRuntimeException e) {
- LOG.warn("Fall back to use GenericRecord instead of POJO for marshalling", e);
+ LOG.warn("Fallback to use GenericRecord instead of POJO for marshalling", e);
}
}
if (schema == null) {
diff --git a/core/camel-core-model/src/generated/resources/org/apache/camel/model/dataformat/parquetAvro.json b/core/camel-core-model/src/generated/resources/org/apache/camel/model/dataformat/parquetAvro.json
index 048b6aa31e3..061701c4db6 100644
--- a/core/camel-core-model/src/generated/resources/org/apache/camel/model/dataformat/parquetAvro.json
+++ b/core/camel-core-model/src/generated/resources/org/apache/camel/model/dataformat/parquetAvro.json
@@ -14,7 +14,7 @@
},
"properties": {
"id": { "index": 0, "kind": "attribute", "displayName": "Id", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The id of this node" },
- "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
+ "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "enum", "javaType": "java.lang.String", "enum": [ "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "ZSTD", "LZ4_RAW" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
"unmarshalType": { "index": 2, "kind": "attribute", "displayName": "Unmarshal Type", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Class to use when (un)marshalling. If omitted, parquet files are converted into Avro's GenericRecords for unmarshalling and input objects are assumed as GenericRecords for marshalling." },
"lazyLoad": { "index": 3, "kind": "attribute", "displayName": "Lazy Load", "required": false, "type": "boolean", "javaType": "java.lang.Boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the unmarshalling should produce an iterator of records or read all the records at once." }
}
diff --git a/core/camel-core-model/src/main/java/org/apache/camel/model/dataformat/ParquetAvroDataFormat.java b/core/camel-core-model/src/main/java/org/apache/camel/model/dataformat/ParquetAvroDataFormat.java
index 402bc819837..fce4152f027 100644
--- a/core/camel-core-model/src/main/java/org/apache/camel/model/dataformat/ParquetAvroDataFormat.java
+++ b/core/camel-core-model/src/main/java/org/apache/camel/model/dataformat/ParquetAvroDataFormat.java
@@ -35,15 +35,12 @@ import org.apache.camel.spi.Metadata;
public class ParquetAvroDataFormat extends DataFormatDefinition {
@XmlAttribute
- @Metadata(defaultValue = "GZIP")
+ @Metadata(defaultValue = "GZIP", enums = "UNCOMPRESSED,SNAPPY,GZIP,LZO,BROTLI,LZ4,ZSTD,LZ4_RAW")
private String compressionCodecName;
-
@XmlTransient
private Class<?> unmarshalType;
-
@XmlAttribute(name = "unmarshalType")
private String unmarshalTypeName;
-
@XmlAttribute
@Metadata(javaType = "java.lang.Boolean")
private String lazyLoad;