You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Suresh Avadhanula (JIRA)" <ji...@apache.org> on 2017/02/02 04:14:51 UTC
[jira] [Comment Edited] (SPARK-19418) Dataset generated java code
fails to compile as java.lang.Long does not accept UTF8String in
constructor
[ https://issues.apache.org/jira/browse/SPARK-19418?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15848890#comment-15848890 ]
Suresh Avadhanula edited comment on SPARK-19418 at 2/2/17 4:14 AM:
-------------------------------------------------------------------
Testcase to reproduce the error. ( see attached encodertest.zip )
{code:title=Compile|borderStyle=solid}
$mvn -P local clean install
{code}
{code:title=Run|borderStyle=solid}
$ java -cp target/encodertest-1.0-SNAPSHOT.jar com.example.spark.EncoderTest
{code}
was (Author: avadhanula):
Testcase to reproduce the error.
{code:title=Compile|borderStyle=solid}
$mvn -P local clean install
{code}
{code:title=Run|borderStyle=solid}
$ java -cp target/encodertest-1.0-SNAPSHOT.jar com.example.spark.EncoderTest
{code}
> Dataset generated java code fails to compile as java.lang.Long does not accept UTF8String in constructor
> --------------------------------------------------------------------------------------------------------
>
> Key: SPARK-19418
> URL: https://issues.apache.org/jira/browse/SPARK-19418
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 2.1.0
> Reporter: Suresh Avadhanula
> Attachments: encodertest.zip
>
>
> I have the following in Java spark driver.
> DealerPerson module object is
> {code:title=DealerPerson.java|borderStyle=solid}
> public class DealerPerson
> {
> Long schemaOrgUnitId ;
> List<Person> personList
> }
> {code}
> I populate it using group by as follows.
> {code}
> Dataset<DealerPerson> dps = persondds.groupByKey(new MapFunction<Person, Long>() {
> @Override
> public Long call(Person person) throws Exception {
> return person.getSchemaOrgUnitId();
> }
> }, Encoders.LONG()).
> mapGroups(new MapGroupsFunction<Long, Person, DealerPerson>() {
> @Override
> public DealerPerson call(Long dp, java.util.Iterator<Person> iterator) throws Exception {
> DealerPerson retDp = new DealerPerson();
> retDp.setSchemaOrgUnitId(dp);
> ArrayList<Person> persons = new ArrayList<Person>();
> while (iterator.hasNext())
> persons.add(iterator.next());
> retDp.setPersons(persons);
> return retDp;
> }
> }, Encoders.bean(DealerPerson.class));
> {code}
> The generated code throws compiler exception since UTF8String is java.lang.Long()
> {noformat}
> 7/01/31 20:32:28 INFO TaskSetManager: Starting task 0.0 in stage 5.0 (TID 5, localhost, executor driver, partition 0, PROCESS_LOCAL, 6442 bytes)
> 17/01/31 20:32:28 INFO Executor: Running task 0.0 in stage 5.0 (TID 5)
> 17/01/31 20:32:28 ERROR CodeGenerator: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 56, Column 58: No applicable constructor/method found for actual parameters "org.apache.spark.unsafe.types.UTF8String"; candidates are: "java.lang.Long(long)", "java.lang.Long(java.lang.String)"
> /* 001 */ public java.lang.Object generate(Object[] references) {
> /* 002 */ return new SpecificSafeProjection(references);
> /* 003 */ }
> /* 004 */
> /* 005 */ class SpecificSafeProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection {
> /* 006 */
> /* 007 */ private Object[] references;
> /* 008 */ private InternalRow mutableRow;
> /* 009 */ private com.xtime.spark.model.Person javaBean;
> /* 010 */ private boolean resultIsNull;
> /* 011 */ private UTF8String argValue;
> /* 012 */ private boolean resultIsNull1;
> /* 013 */ private UTF8String argValue1;
> /* 014 */ private boolean resultIsNull2;
> /* 015 */ private UTF8String argValue2;
> /* 016 */ private boolean resultIsNull3;
> /* 017 */ private UTF8String argValue3;
> /* 018 */ private boolean resultIsNull4;
> /* 019 */ private UTF8String argValue4;
> /* 020 */
> /* 021 */ public SpecificSafeProjection(Object[] references) {
> /* 022 */ this.references = references;
> /* 023 */ mutableRow = (InternalRow) references[references.length - 1];
> /* 024 */
> /* 025 */
> /* 026 */
> /* 027 */
> /* 028 */
> /* 029 */
> /* 030 */
> /* 031 */
> /* 032 */
> /* 033 */
> /* 034 */
> /* 035 */
> /* 036 */ }
> /* 037 */
> /* 038 */ public void initialize(int partitionIndex) {
> /* 039 */
> /* 040 */ }
> /* 041 */
> /* 042 */
> /* 043 */ private void apply_4(InternalRow i) {
> /* 044 */
> /* 045 */
> /* 046 */ resultIsNull1 = false;
> /* 047 */ if (!resultIsNull1) {
> /* 048 */
> /* 049 */ boolean isNull21 = i.isNullAt(17);
> /* 050 */ UTF8String value21 = isNull21 ? null : (i.getUTF8String(17));
> /* 051 */ resultIsNull1 = isNull21;
> /* 052 */ argValue1 = value21;
> /* 053 */ }
> /* 054 */
> /* 055 */
> /* 056 */ final java.lang.Long value20 = resultIsNull1 ? null : new java.lang.Long(argValue1);
> /* 057 */ javaBean.setSchemaOrgUnitId(value20);
> /* 058 */
> /* 059 */
> /* 060 */ resultIsNull2 = false;
> /* 061 */ if (!resultIsNull2) {
> /* 062 */
> /* 063 */ boolean isNull23 = i.isNullAt(0);
> /* 064 */ UTF8String value23 = isNull23 ? null : (i.getUTF8String(0));
> /* 065 */ resultIsNull2 = isNull23;
> /* 066 */ argValue2 = value23;
> /* 067 */ }
> /* 068 */
> /* 069 */
> /* 070 */ final java.lang.Long value22 = resultIsNull2 ? null : new java.lang.Long(argValue2);
> /* 071 */ javaBean.setPersonId(value22);
> /* 072 */
> /* 073 */
> /* 074 */ boolean isNull25 = i.isNullAt(10);
> /* 075 */ UTF8String value25 = isNull25 ? null : (i.getUTF8String(10));
> /* 076 */ boolean isNull24 = true;
> /* 077 */ java.lang.String value24 = null;
> /* 078 */ if (!isNull25) {
> /* 079 */
> /* 080 */ isNull24 = false;
> /* 081 */ if (!isNull24) {
> /* 082 */
> /* 083 */ Object funcResult8 = null;
> /* 084 */ funcResult8 = value25.toString();
> /* 085 */ if (funcResult8 == null) {
> /* 086 */ isNull24 = true;
> /* 087 */ } else {
> /* 088 */ value24 = (java.lang.String) funcResult8;
> /* 089 */ }
> /* 090 */
> /* 091 */ }
> /* 092 */ isNull24 = value24 == null;
> /* 093 */ }
> /* 094 */ javaBean.setCreateMethod(value24);
> /* 095 */
> /* 096 */ }
> /* 097 */
> /* 098 */
> /*
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org