You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@avro.apache.org by "Ryan Skraba (Jira)" <ji...@apache.org> on 2019/11/07 09:38:00 UTC

[jira] [Commented] (AVRO-2602) Updating breaks backward compatibility by throwing AvroTypeException in some cases

    [ https://issues.apache.org/jira/browse/AVRO-2602?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16969087#comment-16969087 ] 

Ryan Skraba commented on AVRO-2602:
-----------------------------------

It looks like the behaviour in 1.7.7 with enums was "tightened up" since 1.8.0.  The test works in 1.7.7 and 1.9.1 if you use:
{code:java}
assertEquals(expectedA, write(schema, new GenericData.EnumSymbol(schema, "A")));{code}

From AVRO-997, it looks like there are many situations where accepting a string representation of the enum causes problems, but the fix was delayed to 1.8 because of the "minor incompatible runtime change"!

> Updating breaks backward compatibility by throwing AvroTypeException in some cases
> ----------------------------------------------------------------------------------
>
>                 Key: AVRO-2602
>                 URL: https://issues.apache.org/jira/browse/AVRO-2602
>             Project: Apache Avro
>          Issue Type: Bug
>    Affects Versions: 1.8.0, 1.8.1, 1.9.0, 1.8.2, 1.9.1
>            Reporter: xia0c
>            Priority: Major
>
> When I try to update Avro from 1.7.7 to the newer version. The following code:
> {code:java}
> 	@Test
> 	public void Demo() throws IOException{
> 		
>         Schema schema = Schema.parse("{\"type\": \"enum\", \"name\": \"MyEnum\", \"symbols\": [\"A\", \"B\", \"C\"]}");
>         Map<String, String> expectedA = new java.util.HashMap();
>         expectedA.put("", "A");
>         assertEquals(expectedA, write(schema, "A"));
> 		
> 	}
> 	
>     private Map<String, String> write(Schema schema, Object datum) throws IOException {
>         DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
>         Map<String, String> out = new java.util.HashMap();
>         KeyValueEncoder encoder = new KeyValueEncoder(schema, out);
>         writer.write(datum, encoder);
>         return out;
>     }
>     
> }
> class KeyValueEncoder extends ParsingEncoder implements Parser.ActionHandler {
> 	  final Parser parser;
> 	  protected BitSet isEmpty = new BitSet();
> 	  private java.util.Map<String, String> out;
> 	  
> 	  KeyValueEncoder(Schema sc, java.util.Map<String, String> out) throws IOException {
> 	    configure(out);
> 	    this.parser =
> 	      new Parser(new JsonGrammarGenerator().generate(sc), this);
> 	  }
> 	  
> 	  public void flush() throws IOException {
> 	    // Do nothing
> 	  }
> 	  public KeyValueEncoder configure(java.util.Map<String, String> newOut) throws IOException {
> 	    this.out = newOut;
> 	    return this;
> 	  }
> 	  
> 	  /////////////////////////////////////////////////////////////////////////////
> 	  @Override
> 	  public void writeNull() throws IOException {
> 	    parser.advance(Symbol.NULL);
> 	  }
> 	  @Override
> 	  public void writeBoolean(boolean b) throws IOException {
> 	    parser.advance(Symbol.BOOLEAN);
> 	    out.put(getKeyPathString(), Boolean.toString(b));
> 	  }
> 	  @Override
> 	  public void writeInt(int n) throws IOException {
> 	    parser.advance(Symbol.INT);
> 	    out.put(getKeyPathString(), Integer.toString(n));
> 	  }
> 	  @Override
> 	  public void writeLong(long n) throws IOException {
> 	    parser.advance(Symbol.LONG);
> 	    out.put(getKeyPathString(), Long.toString(n));
> 	  }
> 	  @Override
> 	  public void writeFloat(float f) throws IOException {
> 	    parser.advance(Symbol.FLOAT);
> 	    out.put(getKeyPathString(), Float.toString(f));
> 	  }
> 	  @Override
> 	  public void writeDouble(double d) throws IOException {
> 	    parser.advance(Symbol.DOUBLE);
> 	    out.put(getKeyPathString(), Double.toString(d));
> 	  }
> 	  @Override
> 	  public void writeString(Utf8 utf8) throws IOException {
> 	    writeString(utf8.toString());
> 	  }
> 	  
> 	  @Override 
> 	  public void writeString(String str) throws IOException {
> 	    parser.advance(Symbol.STRING);
> 	    trace("writeString(" + str + ")");
> 	    if (parser.topSymbol() == Symbol.MAP_KEY_MARKER) {
> 	      parser.advance(Symbol.MAP_KEY_MARKER);
> 	      pushKeyPathComponent(str);
> 	      // out.writeFieldName(str);
> 	    } else {
> 	      out.put(getKeyPathString(), str);
> 	    }
> 	  }
> 	  @Override
> 	  public void writeBytes(ByteBuffer bytes) throws IOException {
> 	    if (bytes.hasArray()) {
> 	      writeBytes(bytes.array(), bytes.position(), bytes.remaining());
> 	    } else {
> 	      byte[] b = new byte[bytes.remaining()];
> 	      for (int i = 0; i < b.length; i++) {
> 	        b[i] = bytes.get();
> 	      }
> 	      writeBytes(b);
> 	    }
> 	  }
> 	  @Override
> 	  public void writeBytes(byte[] bytes, int start, int len) throws IOException {
> 	    parser.advance(Symbol.BYTES);
> 	    writeByteArray(bytes, start, len);
> 	  }
> 	  private void writeByteArray(byte[] bytes, int start, int len) throws IOException {
> 	    out.put(getKeyPathString(), new String(bytes, start, len, "UTF-8"));
> 	  }
> 	  @Override
> 	  public void writeFixed(byte[] bytes, int start, int len) throws IOException {
> 	    throw new IOException("Fixed encoding is not implemented");
> 	  }
> 	  @Override
> 	  public void writeEnum(int e) throws IOException {
> 	    parser.advance(Symbol.ENUM);
> 	    Symbol.EnumLabelsAction top = (Symbol.EnumLabelsAction) parser.popSymbol();
> 	    trace("writeEnum(" + e + " : " + top.getLabel(e) + ")");
> 	    if (e < 0 || e >= top.size) {
> 	      throw new AvroTypeException(
> 	          "Enumeration out of range: max is " +
> 	          top.size + " but received " + e);
> 	    }
> 	    out.put(getKeyPathString(), top.getLabel(e));
> 	  }
> 	  @Override
> 	  public void writeArrayStart() throws IOException {
> 	    throw new IOException("Array encoding is not implemented");
> 	  }
> 	  @Override
> 	  public void writeArrayEnd() throws IOException {
> 	    if (! isEmpty.get(pos)) {
> 	      parser.advance(Symbol.ITEM_END);
> 	    }
> 	    pop();
> 	    parser.advance(Symbol.ARRAY_END);
> 	    // out.writeEndArray();
> 	  }
> 	  @Override
> 	  public void writeMapStart() throws IOException {
> 	    push();
> 	    isEmpty.set(depth());
> 	    parser.advance(Symbol.MAP_START);
> //	     out.writeStartObject();
> 	  }
> 	  @Override
> 	  public void writeMapEnd() throws IOException {
> 	    if (! isEmpty.get(pos)) {
> 	      parser.advance(Symbol.ITEM_END);
> 	      popKeyPathComponent();
> 	    }
> 	    pop();
> 	    parser.advance(Symbol.MAP_END);
> 	    // out.writeEndObject();
> 	  }
> 	  @Override
> 	  public void startItem() throws IOException {
> 	    trace("startItem");
> 	    if (! isEmpty.get(pos)) {
> 	      parser.advance(Symbol.ITEM_END);
> 	      popKeyPathComponent();
> 	    }
> 	    super.startItem();
> 	    isEmpty.clear(depth());
> 	  }
> 	  @Override
> 	  public void writeIndex(int unionIndex) throws IOException {
> 	    parser.advance(Symbol.UNION);
> 	    Symbol.Alternative top = (Symbol.Alternative) parser.popSymbol();
> 	    Symbol symbol = top.getSymbol(unionIndex);
> 	    if (symbol != Symbol.NULL) {
> 	      pushKeyPathComponent(top.getLabel(unionIndex));
> 	      parser.pushSymbol(Symbol.UNION_END);
> 	    }
> 	    parser.pushSymbol(symbol);
> 	  }
> 	  public Symbol doAction(Symbol input, Symbol top) throws IOException {
> 	    if (top instanceof Symbol.FieldAdjustAction) {
> 	      Symbol.FieldAdjustAction fa = (Symbol.FieldAdjustAction) top;
> 	      pushKeyPathComponent(fa.fname);
> 	    } else if (top == Symbol.RECORD_START) {
> 	      // out.writeStartObject();
> 	    } else if (top == Symbol.RECORD_END || top == Symbol.UNION_END) {
> 	      // out.writeEndObject();
> 	    } else if (top == Symbol.FIELD_END) {
> 	      popKeyPathComponent();
> 	    } else {
> 	      throw new AvroTypeException("Unknown action symbol " + top);
> 	    }
> 	    return null;
> 	  }
> 	  /////////////////////////////////////////////////////////////////////////////
> 	  // Key path
> 	  private java.util.ArrayList<String> keyPath = new java.util.ArrayList();
> 	  private void pushKeyPathComponent(String component) {
> 	    keyPath.add(component);
> 	  }
> 	  private void popKeyPathComponent() {
> 	    keyPath.remove(keyPath.size() - 1);
> 	  }
> 	  private String getKeyPathString() {
> 	    return StringUtils.join(keyPath, '|');
> 	  }
> 	  /////////////////////////////////////////////////////////////////////////////
> 	  private void trace(String s) {
> 	    System.out.println(s + ":\t topSymbol=" + parser.topSymbol() + " keyPath=" + getKeyPathString());
> 	  }
> 	
> }
> {code}
> Throws an AvroTypeException error:
> {code:java}
> org.apache.avro.AvroTypeException: Not an enum: A for schema: {"type":"enum","name":"MyEnum","symbols":["A","B","C"]}
>     at org.apache.avro.generic.GenericDatumWriter.writeEnum(GenericDatumWriter.java:218)
>     at org.apache.avro.generic.GenericDatumWriter.writeWithoutConversion(GenericDatumWriter.java:133)
>     at org.apache.avro.generic.GenericDatumWriter.write(GenericDatumWriter.java:82)
>     at org.apache.avro.generic.GenericDatumWriter.write(GenericDatumWriter.java:72)
>     at UTD.SeLab.BBI.BugDetection.TestAvro.write(TestAvro.java:42)
>     at UTD.SeLab.BBI.BugDetection.TestAvro.Demo(TestAvro.java:34)
>     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>     at java.lang.reflect.Method.invoke(Method.java:498)
>     at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
>     at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
>     at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
>     at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
>     at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
>     at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
>     at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
>     at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
>     at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
>     at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
>     at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
>     at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
>     at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
>     at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:86)
>     at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
>     at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:459)
>     at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:678)
>     at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:382)
>     at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:192)
> {code}



--
This message was sent by Atlassian Jira
(v8.3.4#803005)