You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2023/12/26 12:30:34 UTC
(camel) branch main updated: CAMEL-19143: parquet avro dataformat - Add enum for compression levels. (#12607)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 4935f7940c8 CAMEL-19143: parquet avro dataformat - Add enum for compression levels. (#12607)
4935f7940c8 is described below
commit 4935f7940c8ad45823de854068ede99251eb34c2
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Tue Dec 26 13:30:27 2023 +0100
CAMEL-19143: parquet avro dataformat - Add enum for compression levels. (#12607)
CAMEL-19143: parquet avro dataformat - Add enum for compression levels.
---
.../resources/org/apache/camel/catalog/dataformats/parquetAvro.json | 2 +-
.../resources/org/apache/camel/catalog/models/parquetAvro.json | 2 +-
.../org/apache/camel/dataformat/parquet/avro/parquetAvro.json | 2 +-
.../apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java | 4 +---
.../resources/org/apache/camel/model/dataformat/parquetAvro.json | 2 +-
.../org/apache/camel/model/dataformat/ParquetAvroDataFormat.java | 5 +----
.../org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java | 2 +-
.../camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json | 3 ++-
8 files changed, 9 insertions(+), 13 deletions(-)
diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dataformats/parquetAvro.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dataformats/parquetAvro.json
index 45cc3d1217f..70a3b613d67 100644
--- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dataformats/parquetAvro.json
+++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dataformats/parquetAvro.json
@@ -17,7 +17,7 @@
},
"properties": {
"id": { "index": 0, "kind": "attribute", "displayName": "Id", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The id of this node" },
- "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
+ "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "enum", "javaType": "java.lang.String", "enum": [ "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "ZSTD", "LZ4_RAW" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
"unmarshalType": { "index": 2, "kind": "attribute", "displayName": "Unmarshal Type", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Class to use when (un)marshalling. If omitted, parquet files are converted into Avro's GenericRecords for unmarshalling and input objects are assumed as GenericRecords for marshalling." },
"lazyLoad": { "index": 3, "kind": "attribute", "displayName": "Lazy Load", "required": false, "type": "boolean", "javaType": "java.lang.Boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the unmarshalling should produce an iterator of records or read all the records at once." }
}
diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/parquetAvro.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/parquetAvro.json
index 048b6aa31e3..061701c4db6 100644
--- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/parquetAvro.json
+++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/parquetAvro.json
@@ -14,7 +14,7 @@
},
"properties": {
"id": { "index": 0, "kind": "attribute", "displayName": "Id", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The id of this node" },
- "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
+ "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "enum", "javaType": "java.lang.String", "enum": [ "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "ZSTD", "LZ4_RAW" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
"unmarshalType": { "index": 2, "kind": "attribute", "displayName": "Unmarshal Type", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Class to use when (un)marshalling. If omitted, parquet files are converted into Avro's GenericRecords for unmarshalling and input objects are assumed as GenericRecords for marshalling." },
"lazyLoad": { "index": 3, "kind": "attribute", "displayName": "Lazy Load", "required": false, "type": "boolean", "javaType": "java.lang.Boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the unmarshalling should produce an iterator of records or read all the records at once." }
}
diff --git a/components/camel-parquet-avro/src/generated/resources/org/apache/camel/dataformat/parquet/avro/parquetAvro.json b/components/camel-parquet-avro/src/generated/resources/org/apache/camel/dataformat/parquet/avro/parquetAvro.json
index 45cc3d1217f..70a3b613d67 100644
--- a/components/camel-parquet-avro/src/generated/resources/org/apache/camel/dataformat/parquet/avro/parquetAvro.json
+++ b/components/camel-parquet-avro/src/generated/resources/org/apache/camel/dataformat/parquet/avro/parquetAvro.json
@@ -17,7 +17,7 @@
},
"properties": {
"id": { "index": 0, "kind": "attribute", "displayName": "Id", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The id of this node" },
- "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
+ "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "enum", "javaType": "java.lang.String", "enum": [ "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "ZSTD", "LZ4_RAW" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
"unmarshalType": { "index": 2, "kind": "attribute", "displayName": "Unmarshal Type", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Class to use when (un)marshalling. If omitted, parquet files are converted into Avro's GenericRecords for unmarshalling and input objects are assumed as GenericRecords for marshalling." },
"lazyLoad": { "index": 3, "kind": "attribute", "displayName": "Lazy Load", "required": false, "type": "boolean", "javaType": "java.lang.Boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the unmarshalling should produce an iterator of records or read all the records at once." }
}
diff --git a/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java b/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java
index a4809da2bc8..5844f39231d 100644
--- a/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java
+++ b/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java
@@ -55,9 +55,7 @@ public class ParquetAvroDataFormat extends ServiceSupport implements DataFormat,
private static final DefaultUuidGenerator DEFAULT_UUID_GENERATOR = new DefaultUuidGenerator();
private CompressionCodecName compressionCodecName = GZIP;
-
private Class<?> unmarshalType;
-
private boolean lazyLoad;
@Override
@@ -86,7 +84,7 @@ public class ParquetAvroDataFormat extends ServiceSupport implements DataFormat,
schema = ReflectData.AllowNull.get().getSchema(unmarshalType); // generate nullable fields
model = ReflectData.get();
} catch (AvroRuntimeException e) {
- LOG.warn("Fall back to use GenericRecord instead of POJO for marshalling", e);
+ LOG.warn("Fallback to use GenericRecord instead of POJO for marshalling", e);
}
}
if (schema == null) {
diff --git a/core/camel-core-model/src/generated/resources/org/apache/camel/model/dataformat/parquetAvro.json b/core/camel-core-model/src/generated/resources/org/apache/camel/model/dataformat/parquetAvro.json
index 048b6aa31e3..061701c4db6 100644
--- a/core/camel-core-model/src/generated/resources/org/apache/camel/model/dataformat/parquetAvro.json
+++ b/core/camel-core-model/src/generated/resources/org/apache/camel/model/dataformat/parquetAvro.json
@@ -14,7 +14,7 @@
},
"properties": {
"id": { "index": 0, "kind": "attribute", "displayName": "Id", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The id of this node" },
- "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
+ "compressionCodecName": { "index": 1, "kind": "attribute", "displayName": "Compression Codec Name", "required": false, "type": "enum", "javaType": "java.lang.String", "enum": [ "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "ZSTD", "LZ4_RAW" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "GZIP", "description": "Compression codec to use when marshalling." },
"unmarshalType": { "index": 2, "kind": "attribute", "displayName": "Unmarshal Type", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Class to use when (un)marshalling. If omitted, parquet files are converted into Avro's GenericRecords for unmarshalling and input objects are assumed as GenericRecords for marshalling." },
"lazyLoad": { "index": 3, "kind": "attribute", "displayName": "Lazy Load", "required": false, "type": "boolean", "javaType": "java.lang.Boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the unmarshalling should produce an iterator of records or read all the records at once." }
}
diff --git a/core/camel-core-model/src/main/java/org/apache/camel/model/dataformat/ParquetAvroDataFormat.java b/core/camel-core-model/src/main/java/org/apache/camel/model/dataformat/ParquetAvroDataFormat.java
index 402bc819837..fce4152f027 100644
--- a/core/camel-core-model/src/main/java/org/apache/camel/model/dataformat/ParquetAvroDataFormat.java
+++ b/core/camel-core-model/src/main/java/org/apache/camel/model/dataformat/ParquetAvroDataFormat.java
@@ -35,15 +35,12 @@ import org.apache.camel.spi.Metadata;
public class ParquetAvroDataFormat extends DataFormatDefinition {
@XmlAttribute
- @Metadata(defaultValue = "GZIP")
+ @Metadata(defaultValue = "GZIP", enums = "UNCOMPRESSED,SNAPPY,GZIP,LZO,BROTLI,LZ4,ZSTD,LZ4_RAW")
private String compressionCodecName;
-
@XmlTransient
private Class<?> unmarshalType;
-
@XmlAttribute(name = "unmarshalType")
private String unmarshalTypeName;
-
@XmlAttribute
@Metadata(javaType = "java.lang.Boolean")
private String lazyLoad;
diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java
index d7f9c365f24..242300692e7 100644
--- a/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java
+++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java
@@ -10707,7 +10707,7 @@ public final class ModelDeserializers extends YamlDeserializerSupport {
description = "Parquet Avro serialization and de-serialization.",
deprecated = false,
properties = {
- @YamlProperty(name = "compressionCodecName", type = "string", defaultValue = "GZIP", description = "Compression codec to use when marshalling.", displayName = "Compression Codec Name"),
+ @YamlProperty(name = "compressionCodecName", type = "enum:UNCOMPRESSED,SNAPPY,GZIP,LZO,BROTLI,LZ4,ZSTD,LZ4_RAW", defaultValue = "GZIP", description = "Compression codec to use when marshalling.", displayName = "Compression Codec Name"),
@YamlProperty(name = "id", type = "string", description = "The id of this node", displayName = "Id"),
@YamlProperty(name = "lazyLoad", type = "boolean", description = "Whether the unmarshalling should produce an iterator of records or read all the records at once.", displayName = "Lazy Load"),
@YamlProperty(name = "unmarshalType", type = "string", description = "Class to use when (un)marshalling. If omitted, parquet files are converted into Avro's GenericRecords for unmarshalling and input objects are assumed as GenericRecords for marshalling.", displayName = "Unmarshal Type")
diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json
index 112a651e5ab..7e9d6e87219 100644
--- a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json
+++ b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json
@@ -10688,7 +10688,8 @@
"type" : "string",
"title" : "Compression Codec Name",
"description" : "Compression codec to use when marshalling.",
- "default" : "GZIP"
+ "default" : "GZIP",
+ "enum" : [ "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "ZSTD", "LZ4_RAW" ]
},
"id" : {
"type" : "string",