You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2022/12/01 21:00:48 UTC
[beam] branch master updated: Add integer to NUMERIC and BIGNUMERIC conversion support (#24447)
This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new f378533030c Add integer to NUMERIC and BIGNUMERIC conversion support (#24447)
f378533030c is described below
commit f378533030c928ca940634aa4247a1938a771ed3
Author: Siddharth Agrawal <si...@google.com>
AuthorDate: Thu Dec 1 13:00:41 2022 -0800
Add integer to NUMERIC and BIGNUMERIC conversion support (#24447)
---
.../io/gcp/bigquery/TableRowToStorageApiProto.java | 6 ++
.../bigquery/TableRowToStorageApiProtoTest.java | 72 ++++++++++++++++++----
2 files changed, 67 insertions(+), 11 deletions(-)
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java
index cb0b0eaa5e3..cc2dd23b077 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java
@@ -673,6 +673,9 @@ public class TableRowToStorageApiProto {
} else if (value instanceof Double || value instanceof Float) {
return BigDecimalByteStringEncoder.encodeToNumericByteString(
BigDecimal.valueOf(((Number) value).doubleValue()));
+ } else if (value instanceof Short || value instanceof Integer || value instanceof Long) {
+ return BigDecimalByteStringEncoder.encodeToNumericByteString(
+ BigDecimal.valueOf(((Number) value).longValue()));
}
break;
case BIGNUMERIC:
@@ -684,6 +687,9 @@ public class TableRowToStorageApiProto {
} else if (value instanceof Double || value instanceof Float) {
return BigDecimalByteStringEncoder.encodeToBigNumericByteString(
BigDecimal.valueOf(((Number) value).doubleValue()));
+ } else if (value instanceof Short || value instanceof Integer || value instanceof Long) {
+ return BigDecimalByteStringEncoder.encodeToBigNumericByteString(
+ BigDecimal.valueOf(((Number) value).longValue()));
}
break;
case DATETIME:
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java
index 27a772a60ed..1a708ac0537 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java
@@ -79,6 +79,8 @@ public class TableRowToStorageApiProtoTest {
.add(new TableFieldSchema().setType("DATE").setName("dateValue"))
.add(new TableFieldSchema().setType("NUMERIC").setName("numericValue"))
.add(new TableFieldSchema().setType("BIGNUMERIC").setName("bigNumericValue"))
+ .add(new TableFieldSchema().setType("NUMERIC").setName("numericValue2"))
+ .add(new TableFieldSchema().setType("BIGNUMERIC").setName("bigNumericValue2"))
.add(
new TableFieldSchema()
.setType("BYTES")
@@ -116,6 +118,8 @@ public class TableRowToStorageApiProtoTest {
.add(new TableFieldSchema().setType("DATE").setName("dateValue"))
.add(new TableFieldSchema().setType("NUMERIC").setName("numericValue"))
.add(new TableFieldSchema().setType("BIGNUMERIC").setName("bigNumericValue"))
+ .add(new TableFieldSchema().setType("NUMERIC").setName("numericValue2"))
+ .add(new TableFieldSchema().setType("BIGNUMERIC").setName("bigNumericValue2"))
.add(
new TableFieldSchema()
.setType("BYTES")
@@ -244,50 +248,64 @@ public class TableRowToStorageApiProtoTest {
.build())
.addField(
FieldDescriptorProto.newBuilder()
- .setName("arrayvalue")
+ .setName("numericvalue2")
.setNumber(16)
.setType(Type.TYPE_BYTES)
+ .setLabel(Label.LABEL_OPTIONAL)
+ .build())
+ .addField(
+ FieldDescriptorProto.newBuilder()
+ .setName("bignumericvalue2")
+ .setNumber(17)
+ .setType(Type.TYPE_BYTES)
+ .setLabel(Label.LABEL_OPTIONAL)
+ .build())
+ .addField(
+ FieldDescriptorProto.newBuilder()
+ .setName("arrayvalue")
+ .setNumber(18)
+ .setType(Type.TYPE_BYTES)
.setLabel(Label.LABEL_REPEATED)
.build())
.addField(
FieldDescriptorProto.newBuilder()
.setName("timestampisovalue")
- .setNumber(17)
+ .setNumber(19)
.setType(Type.TYPE_INT64)
.setLabel(Label.LABEL_OPTIONAL)
.build())
.addField(
FieldDescriptorProto.newBuilder()
.setName("timestampvaluelong")
- .setNumber(18)
+ .setNumber(20)
.setType(Type.TYPE_INT64)
.setLabel(Label.LABEL_OPTIONAL)
.build())
.addField(
FieldDescriptorProto.newBuilder()
.setName("timestampvaluespace")
- .setNumber(19)
+ .setNumber(21)
.setType(Type.TYPE_INT64)
.setLabel(Label.LABEL_OPTIONAL)
.build())
.addField(
FieldDescriptorProto.newBuilder()
.setName("timestampvaluespacemilli")
- .setNumber(20)
+ .setNumber(22)
.setType(Type.TYPE_INT64)
.setLabel(Label.LABEL_OPTIONAL)
.build())
.addField(
FieldDescriptorProto.newBuilder()
.setName("timestampvaluespacetrailingzero")
- .setNumber(21)
+ .setNumber(23)
.setType(Type.TYPE_INT64)
.setLabel(Label.LABEL_OPTIONAL)
.build())
.addField(
FieldDescriptorProto.newBuilder()
.setName("datetimevaluespace")
- .setNumber(22)
+ .setNumber(24)
.setType(Type.TYPE_INT64)
.setLabel(Label.LABEL_OPTIONAL)
.build())
@@ -395,29 +413,43 @@ public class TableRowToStorageApiProtoTest {
.build())
.addField(
FieldDescriptorProto.newBuilder()
- .setName("arrayvalue")
+ .setName("numericvalue2")
.setNumber(15)
.setType(Type.TYPE_BYTES)
+ .setLabel(Label.LABEL_OPTIONAL)
+ .build())
+ .addField(
+ FieldDescriptorProto.newBuilder()
+ .setName("bignumericvalue2")
+ .setNumber(16)
+ .setType(Type.TYPE_BYTES)
+ .setLabel(Label.LABEL_OPTIONAL)
+ .build())
+ .addField(
+ FieldDescriptorProto.newBuilder()
+ .setName("arrayvalue")
+ .setNumber(17)
+ .setType(Type.TYPE_BYTES)
.setLabel(Label.LABEL_REPEATED)
.build())
.addField(
FieldDescriptorProto.newBuilder()
.setName("timestampisovalue")
- .setNumber(17)
+ .setNumber(18)
.setType(Type.TYPE_INT64)
.setLabel(Label.LABEL_OPTIONAL)
.build())
.addField(
FieldDescriptorProto.newBuilder()
.setName("timestampvaluelong")
- .setNumber(20)
+ .setNumber(19)
.setType(Type.TYPE_INT64)
.setLabel(Label.LABEL_OPTIONAL)
.build())
.addField(
FieldDescriptorProto.newBuilder()
.setName("timestampvaluespace")
- .setNumber(21)
+ .setNumber(20)
.setType(Type.TYPE_INT64)
.setLabel(Label.LABEL_OPTIONAL)
.build())
@@ -578,6 +610,8 @@ public class TableRowToStorageApiProtoTest {
new TableCell().setV("2019-08-16"),
new TableCell().setV("23.4"),
new TableCell().setV("2312345.4"),
+ new TableCell().setV(23),
+ new TableCell().setV(123456789012345678L),
new TableCell().setV(REPEATED_BYTES),
new TableCell().setV("1970-01-01T00:00:00.000+01:00"),
new TableCell().setV("1234567"),
@@ -604,6 +638,8 @@ public class TableRowToStorageApiProtoTest {
.set("dateValue", "2019-08-16")
.set("numericValue", "23.4")
.set("bigNumericValue", "2312345.4")
+ .set("numericValue2", 23)
+ .set("bigNumericValue2", 123456789012345678L)
.set("arrayValue", REPEATED_BYTES)
.set("timestampISOValue", "1970-01-01T00:00:00.000+01:00")
.set("timestampValueLong", "1234567")
@@ -634,6 +670,13 @@ public class TableRowToStorageApiProtoTest {
.put(
"bignumericvalue",
BigDecimalByteStringEncoder.encodeToBigNumericByteString(new BigDecimal("2312345.4")))
+ .put(
+ "numericvalue2",
+ BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("23")))
+ .put(
+ "bignumericvalue2",
+ BigDecimalByteStringEncoder.encodeToBigNumericByteString(
+ new BigDecimal("123456789012345678")))
.put("arrayvalue", EXPECTED_PROTO_REPEATED_BYTES)
.put("timestampisovalue", -3600000000L)
.put("timestampvaluelong", 1234567000L)
@@ -663,6 +706,13 @@ public class TableRowToStorageApiProtoTest {
.put(
"bignumericvalue",
BigDecimalByteStringEncoder.encodeToBigNumericByteString(new BigDecimal("2312345.4")))
+ .put(
+ "numericvalue2",
+ BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("23")))
+ .put(
+ "bignumericvalue2",
+ BigDecimalByteStringEncoder.encodeToBigNumericByteString(
+ new BigDecimal("123456789012345678")))
.put("arrayvalue", EXPECTED_PROTO_REPEATED_BYTES)
.put("timestampisovalue", -3600000000L)
.put("timestampvaluelong", 1234567000L)