You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2024/03/19 16:05:29 UTC
(beam) branch master updated: update confluent dependency version to 7.6.0 (#30638)
This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 764fcda4e9c update confluent dependency version to 7.6.0 (#30638)
764fcda4e9c is described below
commit 764fcda4e9ce1643aea7614d018940a5b93c6489
Author: Maciej Szwaja <ti...@gmail.com>
AuthorDate: Tue Mar 19 17:05:23 2024 +0100
update confluent dependency version to 7.6.0 (#30638)
* update confluent dependency version to 7.6.0
* make avroSchema field non-transient as the Schema class is now Serializable
* remove now unused org.codehaus.jackson package from GcpApiSurfaceTest
* update expected exception class in PubsubClientTest
* fix AvroGenericRecordToStorageApiProtoTest
* move org.tukaani:xz:1.9 dependency to testImplementation
* bring back the tukaani dep to the avroVersions configurations
* trigger the beam_PostCommitJava_Avro_Versions workflow
---
.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json | 4 ++--
buildSrc/build.gradle.kts | 2 +-
.../main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 6 +++---
sdks/java/extensions/avro/build.gradle | 1 +
.../apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java | 2 +-
.../sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java | 3 +--
.../test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 1 -
.../io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java | 5 ++++-
.../apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java | 2 +-
.../java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java | 2 +-
sdks/java/io/kafka/build.gradle | 9 ++++++---
11 files changed, 21 insertions(+), 16 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json b/.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json
index 08c2e40784a..a03c067d2c4 100644
--- a/.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json
+++ b/.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json
@@ -1,3 +1,3 @@
{
- "comment": "Modify this file in a trivial way to cause this test suite to run"
-}
\ No newline at end of file
+ "comment": "Modify this file in a trivial way to cause this test suite to run"
+}
diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts
index bcb2f898f99..cd8aed6d3a6 100644
--- a/buildSrc/build.gradle.kts
+++ b/buildSrc/build.gradle.kts
@@ -45,7 +45,7 @@ dependencies {
implementation("com.github.spotbugs.snom:spotbugs-gradle-plugin:5.0.14")
runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13") // Enable proto code generation
- runtimeOnly("com.github.davidmc24.gradle-avro-plugin:gradle-avro-plugin:0.16.0") // Enable Avro code generation
+ runtimeOnly("com.github.davidmc24.gradle.plugin:gradle-avro-plugin:1.9.1") // Enable Avro code generation
runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:5.6.1") // Enable a code formatting plugin
runtimeOnly("gradle.plugin.com.dorongold.plugins:task-tree:1.5") // Adds a 'taskTree' task to print task dependency tree
runtimeOnly("gradle.plugin.com.github.johnrengelman:shadow:7.1.1") // Enable shading Java dependencies
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index c5d7ed8d571..a03b1940e4a 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -664,8 +664,8 @@ class BeamModulePlugin implements Plugin<Project> {
antlr_runtime : "org.antlr:antlr4-runtime:4.7",
args4j : "args4j:args4j:2.33",
auto_value_annotations : "com.google.auto.value:auto-value-annotations:$autovalue_version",
- avro : "org.apache.avro:avro:1.8.2",
- avro_tests : "org.apache.avro:avro:1.8.2:tests",
+ avro : "org.apache.avro:avro:1.11.3",
+ avro_tests : "org.apache.avro:avro:1.11.3:tests",
aws_java_sdk_cloudwatch : "com.amazonaws:aws-java-sdk-cloudwatch:$aws_java_sdk_version",
aws_java_sdk_core : "com.amazonaws:aws-java-sdk-core:$aws_java_sdk_version",
aws_java_sdk_dynamodb : "com.amazonaws:aws-java-sdk-dynamodb:$aws_java_sdk_version",
@@ -2448,7 +2448,7 @@ class BeamModulePlugin implements Plugin<Project> {
// TODO: Decide whether this should be inlined into the one project that relies on it
// or be left here.
project.ext.applyAvroNature = {
- project.apply plugin: "com.commercehub.gradle.plugin.avro"
+ project.apply plugin: "com.github.davidmc24.gradle.plugin.avro"
// add dependency BeamModulePlugin defined custom tasks
// they are defined only when certain flags are provided (e.g. -Prelease; -Ppublishing, etc)
diff --git a/sdks/java/extensions/avro/build.gradle b/sdks/java/extensions/avro/build.gradle
index ecd3ce867c8..6bf1dfc3f8e 100644
--- a/sdks/java/extensions/avro/build.gradle
+++ b/sdks/java/extensions/avro/build.gradle
@@ -72,6 +72,7 @@ dependencies {
}
testImplementation library.java.avro_tests
testImplementation library.java.junit
+ testImplementation "org.tukaani:xz:1.9" // marked as optional in avro
testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow")
testRuntimeOnly library.java.slf4j_jdk14
avroVersions.each { k,v ->
diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java
index 50be0e6b341..7622132c7e2 100644
--- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java
+++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java
@@ -630,7 +630,7 @@ public class AvroUtils {
}
private static class RowToAvroBytesFn extends SimpleFunction<Row, byte[]> {
- private final transient org.apache.avro.Schema avroSchema;
+ private final org.apache.avro.Schema avroSchema;
private final AvroCoder<GenericRecord> coder;
RowToAvroBytesFn(Schema beamSchema) {
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java
index fc153d6c97d..7141869b228 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java
@@ -262,8 +262,7 @@ public class AvroGenericRecordToStorageApiProto {
elementType.getType() != Schema.Type.UNION,
"Multiple non-null union types are not supported.");
TableFieldSchema unionFieldSchema =
- fieldDescriptorFromAvroField(
- new Schema.Field(field.name(), elementType, field.doc(), field.defaultVal()));
+ fieldDescriptorFromAvroField(new Schema.Field(field.name(), elementType, field.doc()));
builder =
builder
.setType(unionFieldSchema.getType())
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
index e32f45a4382..84eb1298285 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java
@@ -124,7 +124,6 @@ public class GcpApiSurfaceTest {
classesInPackage("javax"),
classesInPackage("org.apache.avro"),
classesInPackage("org.apache.beam"),
- classesInPackage("org.codehaus.jackson"),
classesInPackage("org.joda.time"),
classesInPackage("org.threeten.bp"),
classesInPackage("com.google.gson"));
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java
index f2f5cb69262..ac73a0122d8 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java
@@ -301,7 +301,10 @@ public class AvroGenericRecordToStorageApiProtoTest {
.set(
"decimalValue",
new Conversions.DecimalConversion()
- .toBytes(bd, Schema.create(Schema.Type.NULL), LogicalTypes.decimal(1, 1)))
+ .toBytes(
+ bd,
+ Schema.create(Schema.Type.NULL),
+ LogicalTypes.decimal(bd.precision(), bd.scale())))
.set("dateValue", now)
.set("timestampMicrosValue", now.getMillis() * 1000)
.set("timestampMicrosAsInstantValue", now)
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java
index 0bdecb31a31..c87888134c8 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java
@@ -145,7 +145,7 @@ public class BigQueryAvroUtilsTest {
}
// After a Field is added to a Schema, it is assigned a position, so we can't simply reuse
// the existing Field.
- avroFields.add(new Schema.Field(field.name(), schema, field.doc(), field.defaultValue()));
+ avroFields.add(new Schema.Field(field.name(), schema, field.doc(), field.defaultVal()));
}
Schema avroSchema = Schema.createRecord(avroFields);
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java
index 6461af10a22..895ed35bfb1 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java
@@ -205,7 +205,7 @@ public class PubsubClientTest {
assertThrows(
"null definition should throw an exception",
- NullPointerException.class,
+ SchemaParseException.class,
() ->
PubsubClient.fromPubsubSchema(
com.google.pubsub.v1.Schema.newBuilder().setType(Schema.Type.AVRO).build()));
diff --git a/sdks/java/io/kafka/build.gradle b/sdks/java/io/kafka/build.gradle
index 16f3a39b207..269ddb3f5eb 100644
--- a/sdks/java/io/kafka/build.gradle
+++ b/sdks/java/io/kafka/build.gradle
@@ -29,7 +29,10 @@ provideIntegrationTestingDependencies()
enableJavaPerformanceTesting()
description = "Apache Beam :: SDKs :: Java :: IO :: Kafka"
-ext.summary = "Library to read Kafka topics."
+ext {
+ summary = "Library to read Kafka topics."
+ confluentVersion = "7.6.0"
+}
def kafkaVersions = [
'01103': "0.11.0.3",
@@ -68,14 +71,14 @@ dependencies {
implementation library.java.jackson_annotations
implementation library.java.jackson_databind
implementation "org.springframework:spring-expression:5.3.27"
- implementation ("io.confluent:kafka-avro-serializer:5.3.2") {
+ implementation ("io.confluent:kafka-avro-serializer:${confluentVersion}") {
// zookeeper depends on "spotbugs-annotations:3.1.9" which clashes with current
// "spotbugs-annotations:3.1.12" used in Beam. Not required.
exclude group: "org.apache.zookeeper", module: "zookeeper"
// "kafka-clients" has to be provided since user can use its own version.
exclude group: "org.apache.kafka", module: "kafka-clients"
}
- implementation ("io.confluent:kafka-schema-registry-client:5.3.2") {
+ implementation ("io.confluent:kafka-schema-registry-client:${confluentVersion}") {
// It depends on "spotbugs-annotations:3.1.9" which clashes with current
// "spotbugs-annotations:3.1.12" used in Beam. Not required.
exclude group: "org.apache.zookeeper", module: "zookeeper"