You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by ja...@apache.org on 2020/01/02 16:25:50 UTC
[incubator-daffodil] branch master updated: Fixed incorrect
separator suppression with ock=expression
This is an automated email from the ASF dual-hosted git repository.
jadams pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git
The following commit(s) were added to refs/heads/master by this push:
new 03364e0 Fixed incorrect separator suppression with ock=expression
03364e0 is described below
commit 03364e0a0dc49b13d2d21572069aff2537ea8e84
Author: Josh Adams <ja...@tresys.com>
AuthorDate: Tue Dec 31 15:02:40 2019 -0500
Fixed incorrect separator suppression with ock=expression
A user ran into a bug involving incorrect separator suppresion when a
sequence was using occursCountKind='expression', which should behave as
if separatorSuppressionPolicy='never'.
DAFFODIL-2262
---
.../daffodil/grammar/SequenceGrammarMixin.scala | 2 +-
.../grammar/primitives/SequenceChild.scala | 19 ++-
.../parsers/SeparatedSequenceParsers.scala | 2 +-
.../parsers/UnseparatedSequenceParsers.scala | 2 +-
.../org/apache/daffodil/usertests/Book2.csv | 12 ++
.../org/apache/daffodil/usertests/Book2.dfdl.xsd | 64 ++++++++
.../daffodil/usertests/UserSubmittedTests.tdml | 16 +-
.../daffodil/usertests/test_Book2.expected.xml | 176 +++++++++++++++++++++
.../usertests/TestUserSubmittedTests.scala | 1 +
project/Rat.scala | 1 +
10 files changed, 288 insertions(+), 7 deletions(-)
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/grammar/SequenceGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/grammar/SequenceGrammarMixin.scala
index 699c6c8..330564d 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/grammar/SequenceGrammarMixin.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/grammar/SequenceGrammarMixin.scala
@@ -137,7 +137,7 @@ trait SequenceGrammarMixin
case (e: EB, Ordered__, TrailingStr, Implicit__, ___, UNB) if !e.isLastDeclaredRepresentedInSequence => unboundedPositionalError(e)
case (e: EB, Ordered__, ___________, Fixed_____, ___, `min`) => new RepOrderedExactlyNSequenceChild(this, e, groupIndex, min)
case (e: EB, Ordered__, ___________, Fixed_____, ___, max) => { Assert.invariant(min != max); e.SDE("occursCountKind='fixed' requires minOccurs and maxOccurs to be equal (%d != %d)", min, max) }
- case (e: EB, Ordered__, ___________, Expression, ___, __2) => new RepOrderedExactlyTotalOccursCountSequenceChild(this, e, groupIndex)
+ case (e: EB, Ordered__, ___________, Expression, ___, __2) => new RepOrderedExpressionOccursCountSequenceChild(this, e, groupIndex)
case (e: EB, Ordered__, Never______, Implicit__, ___, UNB) => e.SDE("separatorSuppressionPolicy='never' with occursCountKind='implicit' requires bounded maxOccurs.")
case (e: EB, Ordered__, Never______, Implicit__, ___, max) => new RepOrderedExactlyNSequenceChild(this, e, groupIndex, max)
case (e: EB, Ordered__, Never______, ock /****/ , ___, __2) if (ock ne null) => e.SDE("separatorSuppressionPolicy='never' not allowed in combination with occursCountKind='" + ock + "'.")
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/SequenceChild.scala b/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/SequenceChild.scala
index fd7dd76..f79b1e5 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/SequenceChild.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/SequenceChild.scala
@@ -532,15 +532,28 @@ class RepOrderedExactlyNSequenceChild(sq: SequenceTermBase, e: ElementBase, grou
}
-class RepOrderedExactlyTotalOccursCountSequenceChild(sq: SequenceTermBase, e: ElementBase, groupIndex: Int)
+class RepOrderedExpressionOccursCountSequenceChild(sq: SequenceTermBase, e: ElementBase, groupIndex: Int)
extends RepElementSequenceChild(sq, e, groupIndex) {
lazy val sequenceChildParser: SequenceChildParser = sq.hasSeparator match {
- case true => new RepOrderedExactlyTotalOccursCountSeparatedSequenceChildParser(
+ case true => new RepOrderedExpressionOccursCountSeparatedSequenceChildParser(
childParser, e.occursCountEv, srd, erd, sepParser, sq.separatorPosition, separatedHelper)
- case false => new RepOrderedExactlyTotalOccursCountUnseparatedSequenceChildParser(childParser, e.occursCountEv, srd, erd,
+ case false => new RepOrderedExpressionOccursCountUnseparatedSequenceChildParser(childParser, e.occursCountEv, srd, erd,
unseparatedHelper)
}
+
+ // This class is only used for sequences with occursCountKind="expression",
+ // which means that the separatorSuppressionPolicy is not applicable and the
+ // implied behavior is separatorSuppresionPolicy="never"
+ override lazy val sequenceChildUnparser: SequenceChildUnparser = sq.hasSeparator match {
+ case true => {
+ new RepOrderedSeparatedSequenceChildUnparser(
+ childUnparser, srd, erd, sepUnparser, sq.separatorPosition, SeparatorSuppressionPolicy.Never,
+ zeroLengthDetector, e.isPotentiallyTrailing,
+ true, isPositional, isDeclaredLast)
+ }
+ case false => new RepOrderedUnseparatedSequenceChildUnparser(childUnparser, srd, erd)
+ }
}
class RepOrderedWithMinMaxSequenceChild(sq: SequenceTermBase, e: ElementBase, groupIndex: Int)
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SeparatedSequenceParsers.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SeparatedSequenceParsers.scala
index 8b5c575..05d6269 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SeparatedSequenceParsers.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SeparatedSequenceParsers.scala
@@ -113,7 +113,7 @@ final class RepOrderedExactlyNSeparatedSequenceChildParser(
extends OccursCountExactParser(childParser, srd, erd)
with Separated
-final class RepOrderedExactlyTotalOccursCountSeparatedSequenceChildParser(
+final class RepOrderedExpressionOccursCountSeparatedSequenceChildParser(
childParser: Parser,
ocEv: OccursCountEv,
srd: SequenceRuntimeData,
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/UnseparatedSequenceParsers.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/UnseparatedSequenceParsers.scala
index 9a5f153..af464bb 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/UnseparatedSequenceParsers.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/UnseparatedSequenceParsers.scala
@@ -63,7 +63,7 @@ class RepOrderedExactlyNUnseparatedSequenceChildParser(
extends OccursCountExactParser(childParser, srd, erd)
with Unseparated
-class RepOrderedExactlyTotalOccursCountUnseparatedSequenceChildParser(
+class RepOrderedExpressionOccursCountUnseparatedSequenceChildParser(
childParser: Parser,
ocEv: OccursCountEv,
srd: SequenceRuntimeData,
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/usertests/Book2.csv b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/Book2.csv
new file mode 100644
index 0000000..4419bb2
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/Book2.csv
@@ -0,0 +1,12 @@
+name prefix,name last,name first,name middle,name suffix,nickname,professional title,political title,rank political,rank military,gender
+,$B$2,,$D$2,$E$2,$F$2,$G$2,$H$2,$I$2,$J$2,$K$2
+$A$3,,$C$3,,$E$3,$F$3,$G$3,$H$3,$I$3,$J$3,$K$3
+$A$4,$B$4,,$D$4,,$F$4,$G$4,$H$4,$I$4,$J$4,$K$4
+$A$5,$B$5,$C$5,,$E$5,,$G$5,$H$5,$I$5,$J$5,$K$5
+$A$6,$B$6,$C$6,$D$6,,$F$6,,$H$6,$I$6,$J$6,$K$6
+$A$7,$B$7,$C$7,$D$7,$E$7,,$G$7,,$I$7,$J$7,$K$7
+$A$8,$B$8,$C$8,$D$8,$E$8,$F$8,,$H$8,,$J$8,$K$8
+$A$9,$B$9,$C$9,$D$9,$E$9,$F$9,$G$9,,$I$9,,$K$9
+$A$10,$B$10,$C$10,$D$10,$E$10,$F$10,$G$10,$H$10,,$J$10,
+,$B$11,$C$11,$D$11,$E$11,$F$11,$G$11,$H$11,$I$11,,$K$11
+$A$12,,$C$12,$D$12,$E$12,$F$12,$G$12,$H$12,$I$12,$J$12,
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/usertests/Book2.dfdl.xsd b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/Book2.dfdl.xsd
new file mode 100644
index 0000000..0f7d2a4
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/Book2.dfdl.xsd
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"
+ xmlns:fn="http://www.w3.org/2005/xpath-functions"
+ xmlns:math="http://www.w3.org/2005/xpath-functions/math"
+ xmlns:tns="http://example.com"
+ elementFormDefault="qualified">
+<xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/>
+ <xs:annotation>
+ <xs:appinfo source="http://www.ogf.org/dfdl/">
+ <dfdl:format ref="GeneralFormat"/>
+ <dfdl:defineEscapeScheme name="Quotes">
+ <dfdl:escapeScheme
+ escapeKind="escapeBlock"
+ escapeBlockStart='"'
+ escapeBlockEnd='"'
+ escapeEscapeCharacter="\"
+ extraEscapedCharacters=""
+ generateEscapeBlock="whenNeeded"
+ />
+ </dfdl:defineEscapeScheme>
+ </xs:appinfo>
+ </xs:annotation>
+ <xs:element name="CSV">
+ <xs:complexType>
+ <xs:sequence dfdl:separator="%NL;" dfdl:separatorPosition="infix" dfdl:separatorSuppressionPolicy="trailingEmpty">
+ <xs:element name="header">
+ <xs:complexType>
+ <xs:sequence dfdl:separator="," dfdl:separatorPosition="infix">
+ <xs:element name="title" maxOccurs="unbounded" type="xs:string" dfdl:lengthKind="delimited"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="row" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence dfdl:separator="," dfdl:separatorPosition="infix">
+ <xs:element name="field" minOccurs="11" maxOccurs="unbounded" type="xs:string" dfdl:lengthKind="delimited"
+ dfdl:escapeSchemeRef="Quotes"
+ dfdl:occursCount="{ fn:count(../../header/title) }"
+ dfdl:occursCountKind="expression" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+</xs:schema>
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/usertests/UserSubmittedTests.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/UserSubmittedTests.tdml
index 97bc192..54608ea 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/usertests/UserSubmittedTests.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/UserSubmittedTests.tdml
@@ -116,6 +116,20 @@
<tdml:dfdlInfoset type="file">test_prefix_separator_as_variable.expected.xml
</tdml:dfdlInfoset>
</tdml:infoset>
- </tdml:parserTestCase>
+ </tdml:parserTestCase>
+
+ <!-- DFDL-2262: User test exposed in separator suppression when
+ occursCountKind="expression". Separators and empty elements should never be supressed
+ in this case. -->
+ <tdml:parserTestCase name="test_DFDL_2262" root="CSV" model="Book2.dfdl.xsd" roundTrip="twoPass"
+ description="Demonstrates that separators are not suppressed when OCK='expression'">
+ <tdml:document>
+ <tdml:documentPart type="file">Book2.csv</tdml:documentPart>
+ </tdml:document>
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset type="file">test_Book2.expected.xml</tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
</tdml:testSuite>
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/usertests/test_Book2.expected.xml b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/test_Book2.expected.xml
new file mode 100644
index 0000000..70acc94
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/test_Book2.expected.xml
@@ -0,0 +1,176 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<CSV>
+ <header>
+ <title>name prefix</title>
+ <title>name last</title>
+ <title>name first</title>
+ <title>name middle</title>
+ <title>name suffix</title>
+ <title>nickname</title>
+ <title>professional title</title>
+ <title>political title</title>
+ <title>rank political</title>
+ <title>rank military</title>
+ <title>gender</title>
+ </header>
+ <row>
+ <field></field>
+ <field>$B$2</field>
+ <field></field>
+ <field>$D$2</field>
+ <field>$E$2</field>
+ <field>$F$2</field>
+ <field>$G$2</field>
+ <field>$H$2</field>
+ <field>$I$2</field>
+ <field>$J$2</field>
+ <field>$K$2</field>
+ </row>
+ <row>
+ <field>$A$3</field>
+ <field></field>
+ <field>$C$3</field>
+ <field></field>
+ <field>$E$3</field>
+ <field>$F$3</field>
+ <field>$G$3</field>
+ <field>$H$3</field>
+ <field>$I$3</field>
+ <field>$J$3</field>
+ <field>$K$3</field>
+ </row>
+ <row>
+ <field>$A$4</field>
+ <field>$B$4</field>
+ <field></field>
+ <field>$D$4</field>
+ <field></field>
+ <field>$F$4</field>
+ <field>$G$4</field>
+ <field>$H$4</field>
+ <field>$I$4</field>
+ <field>$J$4</field>
+ <field>$K$4</field>
+ </row>
+ <row>
+ <field>$A$5</field>
+ <field>$B$5</field>
+ <field>$C$5</field>
+ <field></field>
+ <field>$E$5</field>
+ <field></field>
+ <field>$G$5</field>
+ <field>$H$5</field>
+ <field>$I$5</field>
+ <field>$J$5</field>
+ <field>$K$5</field>
+ </row>
+ <row>
+ <field>$A$6</field>
+ <field>$B$6</field>
+ <field>$C$6</field>
+ <field>$D$6</field>
+ <field></field>
+ <field>$F$6</field>
+ <field></field>
+ <field>$H$6</field>
+ <field>$I$6</field>
+ <field>$J$6</field>
+ <field>$K$6</field>
+ </row>
+ <row>
+ <field>$A$7</field>
+ <field>$B$7</field>
+ <field>$C$7</field>
+ <field>$D$7</field>
+ <field>$E$7</field>
+ <field></field>
+ <field>$G$7</field>
+ <field></field>
+ <field>$I$7</field>
+ <field>$J$7</field>
+ <field>$K$7</field>
+ </row>
+ <row>
+ <field>$A$8</field>
+ <field>$B$8</field>
+ <field>$C$8</field>
+ <field>$D$8</field>
+ <field>$E$8</field>
+ <field>$F$8</field>
+ <field></field>
+ <field>$H$8</field>
+ <field></field>
+ <field>$J$8</field>
+ <field>$K$8</field>
+ </row>
+ <row>
+ <field>$A$9</field>
+ <field>$B$9</field>
+ <field>$C$9</field>
+ <field>$D$9</field>
+ <field>$E$9</field>
+ <field>$F$9</field>
+ <field>$G$9</field>
+ <field></field>
+ <field>$I$9</field>
+ <field></field>
+ <field>$K$9</field>
+ </row>
+ <row>
+ <field>$A$10</field>
+ <field>$B$10</field>
+ <field>$C$10</field>
+ <field>$D$10</field>
+ <field>$E$10</field>
+ <field>$F$10</field>
+ <field>$G$10</field>
+ <field>$H$10</field>
+ <field></field>
+ <field>$J$10</field>
+ <field></field>
+ </row>
+ <row>
+ <field></field>
+ <field>$B$11</field>
+ <field>$C$11</field>
+ <field>$D$11</field>
+ <field>$E$11</field>
+ <field>$F$11</field>
+ <field>$G$11</field>
+ <field>$H$11</field>
+ <field>$I$11</field>
+ <field></field>
+ <field>$K$11</field>
+ </row>
+ <row>
+ <field>$A$12</field>
+ <field></field>
+ <field>$C$12</field>
+ <field>$D$12</field>
+ <field>$E$12</field>
+ <field>$F$12</field>
+ <field>$G$12</field>
+ <field>$H$12</field>
+ <field>$I$12</field>
+ <field>$J$12</field>
+ <field></field>
+ </row>
+</CSV>
diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestUserSubmittedTests.scala b/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestUserSubmittedTests.scala
index 73534a1..bd6d6e7 100644
--- a/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestUserSubmittedTests.scala
+++ b/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestUserSubmittedTests.scala
@@ -39,6 +39,7 @@ class TestUserSubmittedTests {
@Test def test_prefix_separator_as_variable() {
runner.runOneTest("test_prefix_separator_as_variable")
}
+ @Test def test_DFDL_2262() { runner.runOneTest("test_DFDL_2262") }
@Test def test_nameDOB_test2_pass() { runner2.runOneTest("nameDOB_test2_pass") }
@Test def test_nameDOB_test2_fail() { runner2.runOneTest("nameDOB_test2_fail") }
diff --git a/project/Rat.scala b/project/Rat.scala
index 847325d..5a91384 100644
--- a/project/Rat.scala
+++ b/project/Rat.scala
@@ -126,6 +126,7 @@ object Rat {
file("daffodil-test/src/test/resources/org/apache/daffodil/section10/representation_properties/encodingError"),
file("daffodil-test/src/test/resources/org/apache/daffodil/section10/representation_properties/littleEndianBit"),
file("daffodil-test/src/test/resources/org/apache/daffodil/usertests/test_prefix_separator_as_variable"),
+ file("daffodil-test/src/test/resources/org/apache/daffodil/usertests/Book2.csv"),
file("daffodil-test/src/test/resources/test space/A BTinyData.tdml.dat"),
file("test-stdLayout/src/test/resources/org1/test-outer-data1.txt"),
file("test-stdLayout/src/test/resources/org2/test-data1.txt"),