You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@daffodil.apache.org by Roger L Costello <co...@mitre.org> on 2024/01/29 16:35:06 UTC

Reusing an element that has DFDL properties results in Daffodil raising a ton of warning messages

Hi Folks,

Dinner time!

My input contains a fixed-length (9) field. The field contains data about dinner time. Dinner time can be expressed using a word, like this:

.../ASAP     /...

Or, it can contain a month-day-hour-minute-timezone value, like this:

.../09190935Z/...

My DFDL schema works fine if I have this choice:

    <xs:element name="DinnerTime"
                type="DinnerTime-complexType"
                dfdl:lengthKind="explicit"
                dfdl:length="9" />

    <xs:complexType name="DinnerTime-complexType">
        <xs:choice>
            <xs:element ref="Word" />
            <xs:element ref="MonthDayTime" />
        </xs:choice>
    </xs:complexType>

But there is actually a third way to express dinner time: day-hour-minute-timezone:

    <xs:complexType name="DinnerTime-complexType">
        <xs:choice>
            <xs:element ref="Word" />
            <xs:element ref="MonthDayTime" />
            <xs:element ref="DayTime" />
        </xs:choice>
    </xs:complexType>

As soon as I add that third way, I start getting a ton of warning messages:

[warn] Schema Definition Warning: DFDL property was ignored: textStandardBase="10"
[warn] Schema Definition Warning: DFDL property was ignored: textNumberPattern="00"
[warn] Schema Definition Warning: DFDL property was ignored: textNumberRounding="pattern"
[warn] Schema Definition Warning: DFDL property was ignored: textNumberRep="standard"
[warn] Schema Definition Warning: DFDL property was ignored: textNumberCheckPolicy="strict"

Why am I getting all those warnings?

Here is my DFDL schema:

<xs:schema xmlns:dfdl=http://www.ogf.org/dfdl/dfdl-1.0/
           xmlns:xs=http://www.w3.org/2001/XMLSchema>

    <xs:include schemaLocation="../default-dfdl-properties/defaults.dfdl.xsd" />

    <xs:annotation>
        <xs:appinfo source=http://www.ogf.org/dfdl/>
            <dfdl:format ref="default-dfdl-properties" />
        </xs:appinfo>
    </xs:annotation>

    <xs:element name="Test">
        <xs:complexType>
            <xs:sequence dfdl:separator="%NL;" dfdl:separatorPosition="infix">
                <xs:element name="Line" maxOccurs="unbounded">
                    <xs:complexType>
                        <xs:sequence dfdl:separator="/" dfdl:separatorPosition="infix">
                            <xs:element name="A" type="xs:string" />
                            <xs:element ref="DinnerTime" />
                            <xs:element name="B" type="xs:string" />
                        </xs:sequence>
                    </xs:complexType>
                </xs:element>
            </xs:sequence>
        </xs:complexType>
    </xs:element>

    <xs:element name="DinnerTime"
                type="DinnerTime-complexType"
                dfdl:lengthKind="explicit"
                dfdl:length="9" />

    <xs:complexType name="DinnerTime-complexType">
        <xs:choice>
            <xs:element ref="Word" />
            <xs:element ref="MonthDayTime" />
            <xs:element ref="DayTime" />
        </xs:choice>
    </xs:complexType>

    <xs:element name="Word"
                type="Word_simpleType"
                dfdl:textTrimKind="padChar"
                dfdl:textPadKind="padChar"
                dfdl:textStringPadCharacter="%SP;"
                dfdl:textStringJustification="left">
        <xs:annotation>
            <xs:appinfo source=http://www.ogf.org/dfdl/>
                <dfdl:assert>{ dfdl:checkConstraints(.) }</dfdl:assert>
            </xs:appinfo>
        </xs:annotation>
    </xs:element>

    <xs:simpleType name="Word_simpleType">
        <xs:restriction base="xs:string">
            <xs:enumeration value="ASAP"/>
        </xs:restriction>
    </xs:simpleType>

    <xs:element name="MonthDayTime">
        <xs:complexType>
            <xs:sequence dfdl:separator="">
                <xs:element ref="Month"/>
                <xs:element ref="Day"/>
                <xs:element ref="Hour"/>
                <xs:element ref="Minute"/>
                <xs:element ref="TimeZone"/>
            </xs:sequence>
        </xs:complexType>
    </xs:element>

    <xs:element name="DayTime">
        <xs:complexType>
            <xs:sequence dfdl:separator="">
                <xs:element ref="Day"/>
                <xs:element ref="Hour"/>
                <xs:element ref="Minute"/>
                <xs:element ref="TimeZone"/>
            </xs:sequence>
        </xs:complexType>
    </xs:element>

    <xs:element name="Month"
                type="xs:unsignedInt"
                dfdl:lengthKind="explicit"
                dfdl:length="2"
                dfdl:textNumberRep="standard"
                dfdl:textNumberCheckPolicy="strict"
                dfdl:textNumberPattern="00"
                dfdl:textStandardGroupingSeparator=","
                dfdl:textStandardDecimalSeparator="."
                dfdl:textStandardBase="10"
                dfdl:textNumberRounding="pattern">
    </xs:element>

    <xs:element name="Day"
                type="xs:unsignedInt"
                dfdl:lengthKind="explicit"
                dfdl:length="2"
                dfdl:textNumberRep="standard"
                dfdl:textNumberCheckPolicy="strict"
                dfdl:textNumberPattern="00"
                dfdl:textStandardGroupingSeparator=","
                dfdl:textStandardDecimalSeparator="."
                dfdl:textStandardBase="10"
                dfdl:textNumberRounding="pattern">
    </xs:element>

    <xs:element name="Hour"
                type="xs:unsignedInt"
                dfdl:lengthKind="explicit"
                dfdl:length="2"
                dfdl:textNumberRep="standard"
                dfdl:textNumberCheckPolicy="strict"
                dfdl:textNumberPattern="00"
                dfdl:textStandardGroupingSeparator=","
                dfdl:textStandardDecimalSeparator="."
                dfdl:textStandardBase="10"
                dfdl:textNumberRounding="pattern">
    </xs:element>

    <xs:element name="Minute"
                type="xs:unsignedInt"
                dfdl:lengthKind="explicit"
                dfdl:length="2"
                dfdl:textNumberRep="standard"
                dfdl:textNumberCheckPolicy="strict"
                dfdl:textNumberPattern="00"
                dfdl:textStandardGroupingSeparator=","
                dfdl:textStandardDecimalSeparator="."
                dfdl:textStandardBase="10"
                dfdl:textNumberRounding="pattern">
    </xs:element>

    <xs:element name="TimeZone"
                type="TimeZone_simpleType"
                dfdl:lengthKind="explicit"
                dfdl:length="1">
        <xs:annotation>
            <xs:appinfo source=http://www.ogf.org/dfdl/>
                <dfdl:assert>{ dfdl:checkConstraints(.) }</dfdl:assert>
            </xs:appinfo>
        </xs:annotation>
    </xs:element>

    <xs:simpleType name="TimeZone_simpleType">
        <xs:restriction base="xs:string">
            <xs:enumeration value="Z"/>
        </xs:restriction>
    </xs:simpleType>

</xs:schema>



Re: Reusing an element that has DFDL properties results in Daffodil raising a ton of warning messages

Posted by Steve Lawrence <sl...@apache.org>.
I think this is just a bug. I'll open a ticket shortly.

One workaround is to use a separate simpleType for your elements and use 
that as the type of your referenced element. For example, change your 
Day element to this:

   <xs:element name="Day" type="DayType" />

   <xs:simpleType name="DayType"
     ... DFDL properties here ... >
     <xs:restriction base="xs:unsignedInt />
   </xs:simpleType>

You can reference Day just like you did before, e.g.

   <xs:element ref="Day" />


For reference, here's what I think is going on:

The schema has multiple references to thes same elements (e.g. two 
references to the Day element). This allows Daffodil to share internal 
parsers. So even though there are two references to the "Day" element, 
we only generate one "Day" sub-parser, and reuse the exact same 
sub-parser in two different places in the final parser we generate. This 
can pretty drastically reduce schema compilation time because we can 
completely skip building a second sub-parser that is identical.

The problem is that we store which properties an element uses on the 
local element that does the referencing, not the element that is 
actually referenced. So we have two differfent lists of used properties.

The first local element generates a sub-parser for the referenced 
element, and that uses all of the properties, and marks the properties 
as used by the first local element.

The second local element figures out that it can share the sub-parser, 
so any properties that are used when that sub-parser is generated do not 
get marked as used on the second local element.

And then when we do our check for unused properties, from the 
perspective of the second local element, some properties were never used 
because it doesn't know the first local element used them and the second 
element skipped all that work.


And I think the workaround works because the referenced element has no 
properties defined directly on it so it looks like there are no unused 
properties. This might actuallt be a separate bug, in that we should be 
considering if the properties defined on the references simple type are 
used or not. This should probably be fixed at the same time as the 
mentioned bug.


On 2024-01-29 11:35 AM, Roger L Costello wrote:
> Hi Folks,
> 
> Dinner time!
> 
> My input contains a fixed-length (9) field. The field contains data 
> about dinner time. Dinner time can be expressed using a word, like this:
> 
> …/ASAP     /…
> 
> Or, it can contain a month-day-hour-minute-timezone value, like this:
> 
> .../09190935Z/...
> 
> My DFDL schema works fine if I have this choice:
> 
> <xs:elementname="DinnerTime"
>                  type="DinnerTime-complexType"
>                  dfdl:lengthKind="explicit"
>                  dfdl:length="9"/>
> 
> <xs:complexTypename="DinnerTime-complexType">
> <xs:choice>
> <xs:elementref="Word"/>
> <xs:elementref="MonthDayTime"/>
> </xs:choice>
> </xs:complexType>
> 
> But there is actually a third way to express dinner time: 
> day-hour-minute-timezone:
> 
> <xs:complexTypename="DinnerTime-complexType">
> <xs:choice>
> <xs:elementref="Word"/>
> <xs:elementref="MonthDayTime"/>
> <xs:elementref="DayTime"/>
> </xs:choice>
> </xs:complexType>
> 
> As soon as I add that third way, I start getting a ton of warning messages:
> 
> [warn] Schema Definition Warning: DFDL property was ignored: 
> textStandardBase="10"
> 
> [warn] Schema Definition Warning: DFDL property was ignored: 
> textNumberPattern="00"
> 
> [warn] Schema Definition Warning: DFDL property was ignored: 
> textNumberRounding="pattern"
> 
> [warn] Schema Definition Warning: DFDL property was ignored: 
> textNumberRep="standard"
> 
> [warn] Schema Definition Warning: DFDL property was ignored: 
> textNumberCheckPolicy="strict"
> 
> Why am I getting all those warnings?
> 
> Here is my DFDL schema:
> 
> <xs:schemaxmlns:dfdl=http://www.ogf.org/dfdl/dfdl-1.0/ 
> <http://www.ogf.org/dfdl/dfdl-1.0/>
> xmlns:xs=http://www.w3.org/2001/XMLSchema 
> <http://www.w3.org/2001/XMLSchema>>
> 
> <xs:includeschemaLocation="../default-dfdl-properties/defaults.dfdl.xsd"/>
> 
> <xs:annotation>
> <xs:appinfosource=http://www.ogf.org/dfdl/ <http://www.ogf.org/dfdl/>>
> <dfdl:formatref="default-dfdl-properties"/>
> </xs:appinfo>
> </xs:annotation>
> 
> <xs:elementname="Test">
> <xs:complexType>
> <xs:sequencedfdl:separator="%NL;"dfdl:separatorPosition="infix">
> <xs:elementname="Line"maxOccurs="unbounded">
> <xs:complexType>
> <xs:sequencedfdl:separator="/"dfdl:separatorPosition="infix">
> <xs:elementname="A"type="xs:string"/>
> <xs:elementref="DinnerTime"/>
> <xs:elementname="B"type="xs:string"/>
> </xs:sequence>
> </xs:complexType>
> </xs:element>
> </xs:sequence>
> </xs:complexType>
> </xs:element>
> 
> <xs:elementname="DinnerTime"
>                  type="DinnerTime-complexType"
>                  dfdl:lengthKind="explicit"
>                  dfdl:length="9"/>
> 
> <xs:complexTypename="DinnerTime-complexType">
> <xs:choice>
> <xs:elementref="Word"/>
> <xs:elementref="MonthDayTime"/>
> <xs:elementref="DayTime"/>
> </xs:choice>
> </xs:complexType>
> 
> <xs:elementname="Word"
>                  type="Word_simpleType"
>                  dfdl:textTrimKind="padChar"
>                  dfdl:textPadKind="padChar"
>                  dfdl:textStringPadCharacter="%SP;"
>                  dfdl:textStringJustification="left">
> <xs:annotation>
> <xs:appinfosource=http://www.ogf.org/dfdl/ <http://www.ogf.org/dfdl/>>
> <dfdl:assert>{ dfdl:checkConstraints(.) }</dfdl:assert>
> </xs:appinfo>
> </xs:annotation>
> </xs:element>
> 
> <xs:simpleTypename="Word_simpleType">
> <xs:restrictionbase="xs:string">
> <xs:enumerationvalue="ASAP"/>
> </xs:restriction>
> </xs:simpleType>
> 
> <xs:elementname="MonthDayTime">
> <xs:complexType>
> <xs:sequencedfdl:separator="">
> <xs:elementref="Month"/>
> <xs:elementref="Day"/>
> <xs:elementref="Hour"/>
> <xs:elementref="Minute"/>
> <xs:elementref="TimeZone"/>
> </xs:sequence>
> </xs:complexType>
> </xs:element>
> 
> <xs:elementname="DayTime">
> <xs:complexType>
> <xs:sequencedfdl:separator="">
> <xs:elementref="Day"/>
> <xs:elementref="Hour"/>
> <xs:elementref="Minute"/>
> <xs:elementref="TimeZone"/>
> </xs:sequence>
> </xs:complexType>
> </xs:element>
> 
> <xs:elementname="Month"
>                  type="xs:unsignedInt"
>                  dfdl:lengthKind="explicit"
>                  dfdl:length="2"
>                  dfdl:textNumberRep="standard"
>                  dfdl:textNumberCheckPolicy="strict"
>                  dfdl:textNumberPattern="00"
>                  dfdl:textStandardGroupingSeparator=","
>                  dfdl:textStandardDecimalSeparator="."
>                  dfdl:textStandardBase="10"
>                  dfdl:textNumberRounding="pattern">
> </xs:element>
> 
> <xs:elementname="Day"
>                  type="xs:unsignedInt"
>                  dfdl:lengthKind="explicit"
>                  dfdl:length="2"
>                  dfdl:textNumberRep="standard"
>                  dfdl:textNumberCheckPolicy="strict"
>                  dfdl:textNumberPattern="00"
>                  dfdl:textStandardGroupingSeparator=","
>                  dfdl:textStandardDecimalSeparator="."
>                  dfdl:textStandardBase="10"
>                  dfdl:textNumberRounding="pattern">
> </xs:element>
> 
> <xs:elementname="Hour"
>                  type="xs:unsignedInt"
>                  dfdl:lengthKind="explicit"
>                  dfdl:length="2"
>                  dfdl:textNumberRep="standard"
>                  dfdl:textNumberCheckPolicy="strict"
>                  dfdl:textNumberPattern="00"
>                  dfdl:textStandardGroupingSeparator=","
>                  dfdl:textStandardDecimalSeparator="."
>                  dfdl:textStandardBase="10"
>                  dfdl:textNumberRounding="pattern">
> </xs:element>
> 
> <xs:elementname="Minute"
>                  type="xs:unsignedInt"
>                  dfdl:lengthKind="explicit"
>                  dfdl:length="2"
>                  dfdl:textNumberRep="standard"
>                  dfdl:textNumberCheckPolicy="strict"
>                  dfdl:textNumberPattern="00"
>                  dfdl:textStandardGroupingSeparator=","
>                  dfdl:textStandardDecimalSeparator="."
>                  dfdl:textStandardBase="10"
>                  dfdl:textNumberRounding="pattern">
> </xs:element>
> 
> <xs:elementname="TimeZone"
>                  type="TimeZone_simpleType"
>                  dfdl:lengthKind="explicit"
>                  dfdl:length="1">
> <xs:annotation>
> <xs:appinfosource=http://www.ogf.org/dfdl/ <http://www.ogf.org/dfdl/>>
> <dfdl:assert>{ dfdl:checkConstraints(.) }</dfdl:assert>
> </xs:appinfo>
> </xs:annotation>
> </xs:element>
> 
> <xs:simpleTypename="TimeZone_simpleType">
> <xs:restrictionbase="xs:string">
> <xs:enumerationvalue="Z"/>
> </xs:restriction>
> </xs:simpleType>
> 
> </xs:schema>
>