You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by ie...@apache.org on 2021/03/04 13:35:10 UTC
[avro] branch master updated (72654bf -> bfe1d03)
This is an automated email from the ASF dual-hosted git repository.
iemejia pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git.
from 72654bf AVRO-3014: Log a warning on ignored logicalType
new e90c0c5 AVRO-2471: Generate specific conversions from LogicalType
new bfe1d03 AVRO-2471: Clean some dead code.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../avro/compiler/specific/SpecificCompiler.java | 36 +++++++++-------------
.../compiler/specific/TestSpecificCompiler.java | 22 ++++++++++++-
.../avro/examples/baseball/FieldTest.java | 2 ++
3 files changed, 37 insertions(+), 23 deletions(-)
[avro] 02/02: AVRO-2471: Clean some dead code.
Posted by ie...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
iemejia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
commit bfe1d03d683d67cdb9a984c02c6b1eb75ffe7689
Author: Ryan Skraba <ry...@skraba.com>
AuthorDate: Wed Aug 5 16:33:49 2020 +0200
AVRO-2471: Clean some dead code.
---
.../avro/compiler/specific/SpecificCompiler.java | 45 ----------------------
.../compiler/specific/TestSpecificCompiler.java | 3 +-
2 files changed, 1 insertion(+), 47 deletions(-)
diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
index 6d363e4..47ff751 100644
--- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
+++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
@@ -331,51 +331,6 @@ public class SpecificCompiler {
return result;
}
- private Set<String> getClassNamesOfPrimitiveFields(Schema schema) {
- Set<String> result = new HashSet<>();
- getClassNamesOfPrimitiveFields(schema, result, new HashSet<>());
- return result;
- }
-
- private void getClassNamesOfPrimitiveFields(Schema schema, Set<String> result, Set<Schema> seenSchemas) {
- if (seenSchemas.contains(schema)) {
- return;
- }
- seenSchemas.add(schema);
- switch (schema.getType()) {
- case RECORD:
- for (Schema.Field field : schema.getFields()) {
- getClassNamesOfPrimitiveFields(field.schema(), result, seenSchemas);
- }
- break;
- case MAP:
- getClassNamesOfPrimitiveFields(schema.getValueType(), result, seenSchemas);
- break;
- case ARRAY:
- getClassNamesOfPrimitiveFields(schema.getElementType(), result, seenSchemas);
- break;
- case UNION:
- for (Schema s : schema.getTypes())
- getClassNamesOfPrimitiveFields(s, result, seenSchemas);
- break;
- case NULL:
- break;
- case ENUM:
- case FIXED:
- case STRING:
- case BYTES:
- case INT:
- case LONG:
- case FLOAT:
- case DOUBLE:
- case BOOLEAN:
- result.add(javaType(schema, true));
- break;
- default:
- throw new RuntimeException("Unknown type: " + schema);
- }
- }
-
private void initializeVelocity() {
this.velocityEngine = new VelocityEngine();
diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java
index fc95454..1ab3b5a 100644
--- a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java
+++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java
@@ -533,8 +533,7 @@ public class TestSpecificCompiler {
SpecificCompiler compiler = createCompiler();
// timestamp-millis and timestamp-micros used to cause collisions when both were
- // present or
- // added as converters (AVRO-2481).
+ // present or added as converters (AVRO-2481).
final Schema tsMillis = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG));
final Schema tsMicros = LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG));
[avro] 01/02: AVRO-2471: Generate specific conversions from
LogicalType
Posted by ie...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
iemejia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
commit e90c0c5cd7740a948fa49172e921bc88573ed699
Author: Ryan Skraba <ry...@skraba.com>
AuthorDate: Fri Jul 31 17:46:07 2020 +0200
AVRO-2471: Generate specific conversions from LogicalType
---
.../avro/compiler/specific/SpecificCompiler.java | 51 +++++++++++++++++++---
.../compiler/specific/TestSpecificCompiler.java | 23 +++++++++-
.../avro/examples/baseball/FieldTest.java | 2 +
3 files changed, 68 insertions(+), 8 deletions(-)
diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
index 8557c6c..6d363e4 100644
--- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
+++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
@@ -281,15 +281,52 @@ public class SpecificCompiler {
}
public Collection<String> getUsedConversionClasses(Schema schema) {
- LinkedHashMap<String, Conversion<?>> classnameToConversion = new LinkedHashMap<>();
- for (Conversion<?> conversion : specificData.getConversions()) {
- classnameToConversion.put(conversion.getConvertedType().getCanonicalName(), conversion);
- }
Collection<String> result = new HashSet<>();
- for (String className : getClassNamesOfPrimitiveFields(schema)) {
- if (classnameToConversion.containsKey(className)) {
- result.add(classnameToConversion.get(className).getClass().getCanonicalName());
+ for (Conversion<?> conversion : getUsedConversions(schema, new HashSet<>(), new HashSet<>())) {
+ result.add(conversion.getClass().getCanonicalName());
+ }
+ return result;
+ }
+
+ private Set<Conversion<?>> getUsedConversions(Schema schema, Set<Conversion<?>> result, Set<Schema> seenSchemas) {
+ if (seenSchemas.contains(schema)) {
+ return result;
+ }
+
+ Conversion<?> conversion = specificData.getConversionFor(LogicalTypes.fromSchemaIgnoreInvalid(schema));
+ if (conversion != null)
+ result.add(conversion);
+
+ seenSchemas.add(schema);
+ switch (schema.getType()) {
+ case RECORD:
+ for (Schema.Field field : schema.getFields()) {
+ getUsedConversions(field.schema(), result, seenSchemas);
}
+ break;
+ case MAP:
+ getUsedConversions(schema.getValueType(), result, seenSchemas);
+ break;
+ case ARRAY:
+ getUsedConversions(schema.getElementType(), result, seenSchemas);
+ break;
+ case UNION:
+ for (Schema s : schema.getTypes())
+ getUsedConversions(s, result, seenSchemas);
+ break;
+ case NULL:
+ case ENUM:
+ case FIXED:
+ case STRING:
+ case BYTES:
+ case INT:
+ case LONG:
+ case FLOAT:
+ case DOUBLE:
+ case BOOLEAN:
+ break;
+ default:
+ throw new RuntimeException("Unknown type: " + schema);
}
return result;
}
diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java
index c0d0d5f..fc95454 100644
--- a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java
+++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java
@@ -17,10 +17,11 @@
*/
package org.apache.avro.compiler.specific;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertThat;
import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.CoreMatchers.equalTo;
@@ -528,6 +529,26 @@ public class TestSpecificCompiler {
}
@Test
+ public void testGetUsedConversionClassesForNullableTimestamps() throws Exception {
+ SpecificCompiler compiler = createCompiler();
+
+ // timestamp-millis and timestamp-micros used to cause collisions when both were
+ // present or
+ // added as converters (AVRO-2481).
+ final Schema tsMillis = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG));
+ final Schema tsMicros = LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG));
+
+ final Collection<String> conversions = compiler.getUsedConversionClasses(SchemaBuilder.record("WithTimestamps")
+ .fields().name("tsMillis").type(tsMillis).noDefault().name("tsMillisOpt").type().unionOf().nullType().and()
+ .type(tsMillis).endUnion().noDefault().name("tsMicros").type(tsMicros).noDefault().name("tsMicrosOpt").type()
+ .unionOf().nullType().and().type(tsMicros).endUnion().noDefault().endRecord());
+
+ Assert.assertEquals(2, conversions.size());
+ assertThat(conversions, hasItem("org.apache.avro.data.TimeConversions.TimestampMillisConversion"));
+ assertThat(conversions, hasItem("org.apache.avro.data.TimeConversions.TimestampMicrosConversion"));
+ }
+
+ @Test
public void testGetUsedConversionClassesForNullableLogicalTypesInNestedRecord() throws Exception {
SpecificCompiler compiler = createCompiler();
diff --git a/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/FieldTest.java b/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/FieldTest.java
index 360b249..7a9bb2a 100644
--- a/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/FieldTest.java
+++ b/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/FieldTest.java
@@ -22,6 +22,8 @@ public class FieldTest extends org.apache.avro.specific.SpecificRecordBase imple
private static SpecificData MODEL$ = new SpecificData();
static {
MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimestampMillisConversion());
+ MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimeMicrosConversion());
+ MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimestampMicrosConversion());
MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimeMillisConversion());
}