You are viewing a plain text version of this content. The canonical link for it is here.

Posted to users@daffodil.apache.org by Roger L Costello <co...@mitre.org> on 2022/12/26 19:18:19 UTC

Another bug in Daffodil

With a 350 MB input file Daffodil gives this error:

org.apache.daffodil.exceptions.Abort: Invariant broken: maybeElem.isDefined
org.apache.daffodil.exceptions.Assert$.abort(Assert.scala:137)
org.apache.daffodil.processors.parsers.NonPositionalGroupSeparatedSequenceChildParseResultHelper.modelGroupSuccessParse
ttemptStatus(SeparatedSequenceChildParseResultHelper.scala:308)
org.apache.daffodil.processors.parsers.ModelGroupSequenceChildParseResultHelper.computeParseAttemptStatus(SequenceChild
arseResultHelper.scala:398)
org.apache.daffodil.processors.parsers.ModelGroupSequenceChildParseResultHelper.computeParseAttemptStatus$(SequenceChil
ParseResultHelper.scala:386)
        at org.apache.daffodil.exceptions.Assert$.abort(Assert.scala:137)
        at org.apache.daffodil.processors.parsers.NonPositionalGroupSeparatedSequenceChildParseResultHelper.modelGroupS
ccessParseAttemptStatus(SeparatedSequenceChildParseResultHelper.scala:308)
        at org.apache.daffodil.processors.parsers.ModelGroupSequenceChildParseResultHelper.computeParseAttemptStatus(Se
uenceChildParseResultHelper.scala:398)
        at org.apache.daffodil.processors.parsers.ModelGroupSequenceChildParseResultHelper.computeParseAttemptStatus$(S
quenceChildParseResultHelper.scala:386)
        at org.apache.daffodil.processors.parsers.NonPositionalGroupSeparatedSequenceChildParseResultHelper.computePars
AttemptStatus(SeparatedSequenceChildParseResultHelper.scala:289)
        at org.apache.daffodil.processors.parsers.InfixPrefixSeparatorHelperMixin.parseOneWithInfixOrPrefixSeparator(Se
aratedParseHelper.scala:139)
        at org.apache.daffodil.processors.parsers.InfixPrefixSeparatorHelperMixin.parseOneWithInfixOrPrefixSeparator$(S
paratedParseHelper.scala:92)
        at org.apache.daffodil.processors.parsers.InfixSeparatorHelper.parseOneWithInfixOrPrefixSeparator(SeparatedPars
Helper.scala:73)
        at org.apache.daffodil.processors.parsers.InfixSeparatorHelper.parseOneWithSeparator(SeparatedParseHelper.scala
84)
        at org.apache.daffodil.processors.parsers.Separated.parseOne(SeparatedSequenceParsers.scala:44)
        at org.apache.daffodil.processors.parsers.Separated.parseOne$(SeparatedSequenceParsers.scala:43)
        at org.apache.daffodil.processors.parsers.ScalarOrderedSeparatedSequenceChildParser.parseOne(SeparatedSequenceP
rsers.scala:52)
        at org.apache.daffodil.processors.parsers.SequenceParserBase.parseOneInstanceWithMaybePoU(SequenceParserBases.s
ala:437)
        at org.apache.daffodil.processors.parsers.SequenceParserBase.parseOneInstance(SequenceParserBases.scala:422)
        at org.apache.daffodil.processors.parsers.SequenceParserBase.parse(SequenceParserBases.scala:265)
        at org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
        at org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
        at org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
        at org.apache.daffodil.processors.parsers.DelimiterStackParser.parse(ElementKindParsers.scala:77)
        at org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
        at org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
        at org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
        at org.apache.daffodil.processors.parsers.DelimiterStackParser.parse(ElementKindParsers.scala:77)
        at org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
        at org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
        at org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
        at org.apache.daffodil.processors.parsers.ElementParserBase.parse(ElementCombinator1.scala:135)
        at org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
        at org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
        at org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
        at org.apache.daffodil.processors.parsers.InfixPrefixSeparatorHelperMixin.parseOneWithInfixOrPrefixSeparator(Se
aratedParseHelper.scala:134)
        at org.apache.daffodil.processors.parsers.InfixPrefixSeparatorHelperMixin.parseOneWithInfixOrPrefixSeparator$(S
paratedParseHelper.scala:92)
        at org.apache.daffodil.processors.parsers.InfixSeparatorHelper.parseOneWithInfixOrPrefixSeparator(SeparatedPars
Helper.scala:73)
        at org.apache.daffodil.processors.parsers.InfixSeparatorHelper.parseOneWithSeparator(SeparatedParseHelper.scala
84)
        at org.apache.daffodil.processors.parsers.Separated.parseOne(SeparatedSequenceParsers.scala:44)
        at org.apache.daffodil.processors.parsers.Separated.parseOne$(SeparatedSequenceParsers.scala:43)
        at org.apache.daffodil.processors.parsers.RepOrderedWithMinMaxSeparatedSequenceChildParser.parseOne(SeparatedSe
uenceParsers.scala:102)
        at org.apache.daffodil.processors.parsers.SequenceParserBase.parseOneInstanceWithMaybePoU(SequenceParserBases.s
ala:437)
        at org.apache.daffodil.processors.parsers.SequenceParserBase.parseOneInstance(SequenceParserBases.scala:417)
        at org.apache.daffodil.processors.parsers.SequenceParserBase.parse(SequenceParserBases.scala:157)
        at org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
        at org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
        at org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
        at org.apache.daffodil.processors.parsers.DelimiterStackParser.parse(ElementKindParsers.scala:77)
        at org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
        at org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
        at org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
        at org.apache.daffodil.processors.parsers.ElementParserBase.parse(ElementCombinator1.scala:135)
        at org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
        at org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
        at org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
        at org.apache.daffodil.processors.DataProcessor.doParse(DataProcessor.scala:503)
        at org.apache.daffodil.processors.DataProcessor.parse(DataProcessor.scala:447)
        at org.apache.daffodil.Main$.$anonfun$run$3(Main.scala:933)
        at org.apache.daffodil.util.Timer$.getTimeResult(Timer.scala:78)
        at org.apache.daffodil.util.Timer$.getResult(Timer.scala:37)
        at org.apache.daffodil.Main$.run(Main.scala:933)
        at org.apache.daffodil.Main$.main(Main.scala:1577)
        at org.apache.daffodil.Main.main(Main.scala)

Re: Another bug in Daffodil

Posted by Mike Beckerle <mb...@apache.org>.

Ok, to clarify, your same schema works with smaller files, but when you up
the data size to 350MB you get this failure.

We do have tunable limits that may need to be enlarged, but you should not
be getting an abort like this due to hitting those limits... if that's even
the issue.

We need to reproduce this, so we would need your schema, your small test
data file, your large test data file.

Is this material something you are willing to share, or is it CUI/FOUO or
otherwise confidential so we need to take other steps?

-mikeb


On Mon, Dec 26, 2022 at 2:18 PM Roger L Costello <co...@mitre.org> wrote:

> With a 350 MB input file Daffodil gives this error:
>
>
>
> org.apache.daffodil.exceptions.Abort: Invariant broken: maybeElem.isDefined
>
> org.apache.daffodil.exceptions.Assert$.abort(Assert.scala:137)
>
>
> org.apache.daffodil.processors.parsers.NonPositionalGroupSeparatedSequenceChildParseResultHelper.modelGroupSuccessParse
>
> ttemptStatus(SeparatedSequenceChildParseResultHelper.scala:308)
>
>
> org.apache.daffodil.processors.parsers.ModelGroupSequenceChildParseResultHelper.computeParseAttemptStatus(SequenceChild
>
> arseResultHelper.scala:398)
>
>
> org.apache.daffodil.processors.parsers.ModelGroupSequenceChildParseResultHelper.computeParseAttemptStatus$(SequenceChil
>
> ParseResultHelper.scala:386)
>
>         at org.apache.daffodil.exceptions.Assert$.abort(Assert.scala:137)
>
>         at
> org.apache.daffodil.processors.parsers.NonPositionalGroupSeparatedSequenceChildParseResultHelper.modelGroupS
>
> ccessParseAttemptStatus(SeparatedSequenceChildParseResultHelper.scala:308)
>
>         at
> org.apache.daffodil.processors.parsers.ModelGroupSequenceChildParseResultHelper.computeParseAttemptStatus(Se
>
> uenceChildParseResultHelper.scala:398)
>
>         at
> org.apache.daffodil.processors.parsers.ModelGroupSequenceChildParseResultHelper.computeParseAttemptStatus$(S
>
> quenceChildParseResultHelper.scala:386)
>
>         at
> org.apache.daffodil.processors.parsers.NonPositionalGroupSeparatedSequenceChildParseResultHelper.computePars
>
> AttemptStatus(SeparatedSequenceChildParseResultHelper.scala:289)
>
>         at
> org.apache.daffodil.processors.parsers.InfixPrefixSeparatorHelperMixin.parseOneWithInfixOrPrefixSeparator(Se
>
> aratedParseHelper.scala:139)
>
>         at
> org.apache.daffodil.processors.parsers.InfixPrefixSeparatorHelperMixin.parseOneWithInfixOrPrefixSeparator$(S
>
> paratedParseHelper.scala:92)
>
>         at
> org.apache.daffodil.processors.parsers.InfixSeparatorHelper.parseOneWithInfixOrPrefixSeparator(SeparatedPars
>
> Helper.scala:73)
>
>         at
> org.apache.daffodil.processors.parsers.InfixSeparatorHelper.parseOneWithSeparator(SeparatedParseHelper.scala
>
> 84)
>
>         at
> org.apache.daffodil.processors.parsers.Separated.parseOne(SeparatedSequenceParsers.scala:44)
>
>         at
> org.apache.daffodil.processors.parsers.Separated.parseOne$(SeparatedSequenceParsers.scala:43)
>
>         at
> org.apache.daffodil.processors.parsers.ScalarOrderedSeparatedSequenceChildParser.parseOne(SeparatedSequenceP
>
> rsers.scala:52)
>
>         at
> org.apache.daffodil.processors.parsers.SequenceParserBase.parseOneInstanceWithMaybePoU(SequenceParserBases.s
>
> ala:437)
>
>         at
> org.apache.daffodil.processors.parsers.SequenceParserBase.parseOneInstance(SequenceParserBases.scala:422)
>
>         at
> org.apache.daffodil.processors.parsers.SequenceParserBase.parse(SequenceParserBases.scala:265)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
>
>         at
> org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
>
>         at
> org.apache.daffodil.processors.parsers.DelimiterStackParser.parse(ElementKindParsers.scala:77)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
>
>         at
> org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
>
>         at
> org.apache.daffodil.processors.parsers.DelimiterStackParser.parse(ElementKindParsers.scala:77)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
>
>         at
> org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
>
>         at
> org.apache.daffodil.processors.parsers.ElementParserBase.parse(ElementCombinator1.scala:135)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
>
>         at
> org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
>
>         at
> org.apache.daffodil.processors.parsers.InfixPrefixSeparatorHelperMixin.parseOneWithInfixOrPrefixSeparator(Se
>
> aratedParseHelper.scala:134)
>
>         at
> org.apache.daffodil.processors.parsers.InfixPrefixSeparatorHelperMixin.parseOneWithInfixOrPrefixSeparator$(S
>
> paratedParseHelper.scala:92)
>
>         at
> org.apache.daffodil.processors.parsers.InfixSeparatorHelper.parseOneWithInfixOrPrefixSeparator(SeparatedPars
>
> Helper.scala:73)
>
>         at
> org.apache.daffodil.processors.parsers.InfixSeparatorHelper.parseOneWithSeparator(SeparatedParseHelper.scala
>
> 84)
>
>         at
> org.apache.daffodil.processors.parsers.Separated.parseOne(SeparatedSequenceParsers.scala:44)
>
>         at
> org.apache.daffodil.processors.parsers.Separated.parseOne$(SeparatedSequenceParsers.scala:43)
>
>         at
> org.apache.daffodil.processors.parsers.RepOrderedWithMinMaxSeparatedSequenceChildParser.parseOne(SeparatedSe
>
> uenceParsers.scala:102)
>
>         at
> org.apache.daffodil.processors.parsers.SequenceParserBase.parseOneInstanceWithMaybePoU(SequenceParserBases.s
>
> ala:437)
>
>         at
> org.apache.daffodil.processors.parsers.SequenceParserBase.parseOneInstance(SequenceParserBases.scala:417)
>
>         at
> org.apache.daffodil.processors.parsers.SequenceParserBase.parse(SequenceParserBases.scala:157)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
>
>         at
> org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
>
>         at
> org.apache.daffodil.processors.parsers.DelimiterStackParser.parse(ElementKindParsers.scala:77)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
>
>         at
> org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
>
>         at
> org.apache.daffodil.processors.parsers.ElementParserBase.parse(ElementCombinator1.scala:135)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1(Parser.scala:82)
>
>         at
> org.apache.daffodil.processors.parsers.Parser.parse1$(Parser.scala:76)
>
>         at
> org.apache.daffodil.processors.parsers.CombinatorParser.parse1(Parser.scala:161)
>
>         at
> org.apache.daffodil.processors.DataProcessor.doParse(DataProcessor.scala:503)
>
>         at
> org.apache.daffodil.processors.DataProcessor.parse(DataProcessor.scala:447)
>
>         at org.apache.daffodil.Main$.$anonfun$run$3(Main.scala:933)
>
>         at org.apache.daffodil.util.Timer$.getTimeResult(Timer.scala:78)
>
>         at org.apache.daffodil.util.Timer$.getResult(Timer.scala:37)
>
>         at org.apache.daffodil.Main$.run(Main.scala:933)
>
>         at org.apache.daffodil.Main$.main(Main.scala:1577)
>
>         at org.apache.daffodil.Main.main(Main.scala)
>