You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by mb...@apache.org on 2019/05/28 15:56:14 UTC
[incubator-daffodil] branch master updated: Added a few new tests
that verify specific sequence behaviors.
This is an automated email from the ASF dual-hosted git repository.
mbeckerle pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git
The following commit(s) were added to refs/heads/master by this push:
new bb4d965 Added a few new tests that verify specific sequence behaviors.
bb4d965 is described below
commit bb4d96556269d04faf7c5029451f57b0fea8a7cb
Author: Michael Beckerle <mb...@tresys.com>
AuthorDate: Fri May 24 17:07:26 2019 -0400
Added a few new tests that verify specific sequence behaviors.
Fixed bugs with not pushing/popping the array stack properly.
This enabled agnosc_remedy, army_drrs, and uscg_ucop schemas to work
again.
DAFFODIL-2135
---
.../scala/org/apache/daffodil/xml/XMLUtils.scala | 23 +--
.../unparsers/SeparatedSequenceUnparsers.scala | 15 ++
.../unparsers/SequenceChildUnparsers.scala | 12 +-
.../unparsers/UnseparatedSequenceUnparsers.scala | 2 +
.../daffodil/processors/unparsers/UState.scala | 1 -
.../test-suite/tresys-contributed/nilled.tdml | 171 +++++++++++++++++++++
.../test-suite/tresys-contributed/unseparated.tdml | 75 +++++++++
.../scala/org/apache/daffodil/TestNilled.scala | 32 ++++
.../org/apache/daffodil/TestUnseparated.scala | 33 ++++
9 files changed, 343 insertions(+), 21 deletions(-)
diff --git a/daffodil-lib/src/main/scala/org/apache/daffodil/xml/XMLUtils.scala b/daffodil-lib/src/main/scala/org/apache/daffodil/xml/XMLUtils.scala
index 723d511..91f07dc 100644
--- a/daffodil-lib/src/main/scala/org/apache/daffodil/xml/XMLUtils.scala
+++ b/daffodil-lib/src/main/scala/org/apache/daffodil/xml/XMLUtils.scala
@@ -387,7 +387,8 @@ object XMLUtils {
val dafintURI = DAFFODIL_INTERNAL_NAMESPACE
val DFDL_SIMPLE_BUILT_IN_TYPES =
- List("string",
+ List(
+ "string",
"float",
"double",
"decimal",
@@ -727,7 +728,7 @@ object XMLUtils {
/**
* Prepares an XML node for diff comparison
- **/
+ */
private def prepareForDiffComparison(n: Node): Node = {
val noComments = removeComments(n)
val noPCData = convertPCDataToText(noComments)
@@ -745,15 +746,18 @@ object XMLUtils {
if (diffs.length > 0) {
throw new XMLDifferenceException("""
Comparison failed.
-Expected
+Expected (attributes stripped)
%s
-Actual
+Actual (attributes stripped)
%s
Differences were (path, expected, actual):
+%s
+Actual with Attributes (needed for unparse)
%s""".format(
- removeAttributes(expected).toString,
- removeAttributes(actual).toString,
- diffs.map { _.toString }.mkString("- ", "\n- ", "\n")))
+ removeAttributes(expected).toString,
+ removeAttributes(actual).toString,
+ diffs.map { _.toString }.mkString("- ", "\n- ", "\n"),
+ actual))
}
}
@@ -881,7 +885,7 @@ Differences were (path, expected, actual):
def computeTextDiff(
zPath: String,
tA: Text,
- tB:Text,
+ tB: Text,
maybeType: Option[String]): Seq[(String, String, String)] = {
val dataA = tA.toString
@@ -967,7 +971,8 @@ Differences were (path, expected, actual):
tmpSchemaFile
}
- def convertInputStreamToTempFile(is: java.io.InputStream,
+ def convertInputStreamToTempFile(
+ is: java.io.InputStream,
tmpDir: File,
nameHint: String,
suffix: String) = {
diff --git a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/SeparatedSequenceUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/SeparatedSequenceUnparsers.scala
index 7730414..43c5912 100644
--- a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/SeparatedSequenceUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/SeparatedSequenceUnparsers.scala
@@ -255,6 +255,7 @@ class OrderedSeparatedSequenceUnparser(
val zlDetector = childUnparser.zeroLengthDetector
childUnparser match {
case unparser: RepOrderedSeparatedSequenceChildUnparser => {
+ state.arrayIndexStack.push(1L)
val erd = unparser.erd
var numOccurrences = 0
val maxReps = unparser.maxRepeats(state)
@@ -293,6 +294,15 @@ class OrderedSeparatedSequenceUnparser(
doUnparser = unparser.shouldDoUnparser(unparser, state)
doUnparser
}) {
+ //
+ // These are so we can check invariants on these stacks being
+ // pushed and popped reliably, and incremented only once
+ //
+ val arrayIndexBefore = state.arrayPos
+ val arrayIndexStackDepthBefore = state.arrayIndexStack.length
+ val groupIndexBefore = state.groupPos
+ val groupIndexStackDepthBefore = state.groupIndexStack.length
+
Assert.invariant(erd.isRepresented) // since this is an array, can't have inputValueCalc
if (isArr) if (state.dataProc.isDefined) state.dataProc.get.beforeRepetition(state, this)
@@ -311,11 +321,15 @@ class OrderedSeparatedSequenceUnparser(
onlySeparatorFlag = false)
}
numOccurrences += 1
+ Assert.invariant(state.arrayIndexStack.length == arrayIndexStackDepthBefore)
state.moveOverOneArrayIndexOnly()
+ Assert.invariant(state.arrayPos == arrayIndexBefore + 1)
// System.err.println("Finished unparse of occurrence of %s. Array Index Stack is: %s".format(
// erd.namedQName, state.arrayIndexStack))
+ Assert.invariant(state.groupIndexStack.length == groupIndexStackDepthBefore)
state.moveOverOneGroupIndexOnly() // array elements are always represented.
+ Assert.invariant(state.groupPos == groupIndexBefore + 1)
if (isArr) if (state.dataProc.isDefined) state.dataProc.get.afterRepetition(state, this)
@@ -352,6 +366,7 @@ class OrderedSeparatedSequenceUnparser(
// no event (state.inspect returned false)
Assert.invariantFailed("No event for unparsing.")
}
+ state.arrayIndexStack.pop()
}
case scalarUnparser => trd match {
case erd: ElementRuntimeData => {
diff --git a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/SequenceChildUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/SequenceChildUnparsers.scala
index ddc8721..8777709 100644
--- a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/SequenceChildUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/SequenceChildUnparsers.scala
@@ -84,15 +84,6 @@ abstract class RepeatingChildUnparser(
* event.
*/
final def startArrayOrOptional(state: UState): Unit = {
- //
- // The Infoset events don't create an array start/end event for a
- // truly optional (0..1) element.
- //
- // But an expression might still refer to dfdl:occursIndex() and we want that
- // to be 1, not wherever an enclosing parent array is.
- //
- state.arrayIndexStack.push(1L) // one-based indexing
-
val ev = state.inspectAccessor
if (ev.erd.isArray) {
// only pull start array event for a true array, not an optional.
@@ -116,8 +107,7 @@ abstract class RepeatingChildUnparser(
}
}
- // However, we pop the arrayIndexStack for either true arrays or optionals.
- val actualOccurs = state.arrayIndexStack.pop()
+ val actualOccurs = state.arrayPos
Assert.invariant(state.processorStatus eq Success)
diff --git a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/UnseparatedSequenceUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/UnseparatedSequenceUnparsers.scala
index ae44f98..d120f2c 100644
--- a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/UnseparatedSequenceUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/UnseparatedSequenceUnparsers.scala
@@ -90,6 +90,7 @@ class OrderedUnseparatedSequenceUnparser(rd: SequenceRuntimeData, childUnparsers
//
childUnparser match {
case unparser: RepeatingChildUnparser => {
+ state.arrayIndexStack.push(1L)
val erd = unparser.erd
var numOccurrences = 0
val maxReps = unparser.maxRepeats(state)
@@ -159,6 +160,7 @@ class OrderedUnseparatedSequenceUnparser(rd: SequenceRuntimeData, childUnparsers
// no event (state.inspect returned false)
Assert.invariantFailed("No event for unparing.")
}
+ state.arrayIndexStack.pop()
}
//
case scalarUnparser => {
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/unparsers/UState.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/unparsers/UState.scala
index 57753e7..08ded35 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/unparsers/UState.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/unparsers/UState.scala
@@ -627,4 +627,3 @@ object UState {
newState
}
}
-
diff --git a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/nilled.tdml b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/nilled.tdml
new file mode 100644
index 0000000..9935840
--- /dev/null
+++ b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/nilled.tdml
@@ -0,0 +1,171 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<tdml:testSuite
+ xmlns:tdml="http://www.ibm.com/xmlns/dfdl/testData"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"
+ xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ xmlns:ex="http://example.com"
+ defaultImplementations="ibm daffodil"
+ defaultRoundTrip="onePass"
+ defaultConfig="cfg">
+
+ <tdml:defineConfig name="cfg">
+ <daf:tunables>
+ <daf:suppressSchemaDefinitionWarnings>
+ unsupportedAttributeFormDefault
+ encodingErrorPolicyError
+ </daf:suppressSchemaDefinitionWarnings>
+ </daf:tunables>
+ </tdml:defineConfig>
+
+ <tdml:defineConfig name="cfg_noEmptyElements">
+ <daf:tunables>
+ <daf:defaultEmptyElementParsePolicy>treatAsMissing</daf:defaultEmptyElementParsePolicy>
+ <daf:suppressSchemaDefinitionWarnings>
+ unsupportedAttributeFormDefault
+ encodingErrorPolicyError
+ </daf:suppressSchemaDefinitionWarnings>
+ </daf:tunables>
+ </tdml:defineConfig>
+
+ <tdml:defineSchema name="s1" elementFormDefault="qualified">
+ <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd" />
+
+ <dfdl:format ref="ex:GeneralFormatPortable" lengthKind="delimited"
+ occursCountKind="implicit" nilValue="%ES;"
+ nilValueDelimiterPolicy="none"
+ nilKind="literalValue"
+ separatorSuppressionPolicy="anyEmpty"/>
+
+ <xs:element name="r1">
+ <xs:complexType>
+ <xs:sequence dfdl:separator="%CR;%CR;%LF;">
+ <xs:element ref="ex:set" maxOccurs="100"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="set">
+ <xs:complexType>
+ <xs:choice>
+ <xs:element ref="ex:r" />
+ <xs:element ref="ex:s" />
+ </xs:choice>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="r" dfdl:initiator="R/">
+ <xs:complexType>
+ <xs:sequence dfdl:separator="/ %CR;%CR;%LF;/" dfdl:separatorSuppressionPolicy="trailingEmpty">
+ <xs:element ref="ex:e" />
+ <xs:element ref="ex:f" minOccurs="0" maxOccurs="1" />
+ <xs:element ref="ex:g" minOccurs="0" maxOccurs="1" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="s" dfdl:initiator="S/">
+ <xs:complexType>
+ <xs:sequence dfdl:separator="/ %CR;%CR;%LF;/" dfdl:separatorSuppressionPolicy="trailingEmpty">
+ <xs:element ref="ex:e" />
+ <xs:element ref="ex:f" minOccurs="0" maxOccurs="1" />
+ <xs:element ref="ex:g" minOccurs="0" maxOccurs="1" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="e">
+ <xs:simpleType>
+ <xs:restriction base="xs:int">
+ <xs:minInclusive value="10" />
+ <xs:maxInclusive value="99" />
+ </xs:restriction>
+ </xs:simpleType>
+ </xs:element>
+
+ <xs:element name="f" nillable="true">
+ <xs:annotation>
+ <xs:appinfo source="http://www.ogf.org/dfdl/">
+ <dfdl:assert test="{ dfdl:checkConstraints(.) }" message="must be between 4 and 5 characters long" />
+ </xs:appinfo>
+ </xs:annotation>
+ <xs:simpleType>
+ <xs:restriction base="ex:UppercaseAlphaOrNumeric">
+ <xs:minLength value="4" />
+ <xs:maxLength value="5" />
+ </xs:restriction>
+ </xs:simpleType>
+ </xs:element>
+
+ <xs:simpleType name="UppercaseAlphaOrNumeric">
+ <xs:restriction base="xs:string">
+ <xs:pattern value="[A-Z0-9]+" />
+ </xs:restriction>
+ </xs:simpleType>
+
+ <xs:element name="g" nillable="true">
+ <xs:annotation>
+ <xs:appinfo source="http://www.ogf.org/dfdl/">
+ <dfdl:assert test="{ dfdl:checkConstraints(.) }" message="must be exactly 1 digit" />
+ </xs:appinfo>
+ </xs:annotation>
+ <xs:simpleType>
+ <xs:restriction base="xs:int">
+ <xs:minInclusive value="0" />
+ <xs:maxInclusive value="9" />
+ </xs:restriction>
+ </xs:simpleType>
+ </xs:element>
+
+ </tdml:defineSchema>
+
+ <tdml:parserTestCase name="nilled_ES_optional" root="r1" model="s1"
+ roundTrip="twoPass">
+
+ <tdml:document>
+ <tdml:documentPart type="text" replaceDFDLEntities="true">R/10%CR;%CR;%LF;//4%CR;%CR;%LF;S/11/ABCD/5</tdml:documentPart>
+ </tdml:document>
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <r1 xmlns="http://example.com">
+ <set>
+ <r>
+ <e>10</e>
+ <f xsi:nil="true" />
+ <g>4</g>
+ </r>
+ </set>
+ <set>
+ <s>
+ <e>11</e>
+ <f>ABCD</f>
+ <g>5</g>
+ </s>
+ </set>
+ </r1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ </tdml:parserTestCase>
+
+
+</tdml:testSuite>
diff --git a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/unseparated.tdml b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/unseparated.tdml
new file mode 100644
index 0000000..e520ba0
--- /dev/null
+++ b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/unseparated.tdml
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<tdml:testSuite suiteName="uscg_ucop_lh.dfdl" description="USCG_UCOP_LH Tests"
+ xmlns:tdml="http://www.ibm.com/xmlns/dfdl/testData"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"
+ xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ xmlns:ex="http://example.com"
+ defaultRoundTrip="onePass"
+ defaultConfig="cfg">
+
+ <tdml:defineConfig name="cfg">
+ <daf:tunables>
+ <daf:suppressSchemaDefinitionWarnings>
+ unsupportedAttributeFormDefault
+ encodingErrorPolicyError
+ </daf:suppressSchemaDefinitionWarnings>
+ </daf:tunables>
+ </tdml:defineConfig>
+
+ <tdml:defineConfig name="cfg_noEmptyElements">
+ <daf:tunables>
+ <daf:defaultEmptyElementParsePolicy>treatAsMissing</daf:defaultEmptyElementParsePolicy>
+ <daf:suppressSchemaDefinitionWarnings>
+ unsupportedAttributeFormDefault
+ encodingErrorPolicyError
+ </daf:suppressSchemaDefinitionWarnings>
+ </daf:tunables>
+ </tdml:defineConfig>
+
+ <tdml:defineSchema name="s1" elementFormDefault="unqualified">
+ <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd" />
+ <dfdl:format ref="ex:GeneralFormatPortable" />
+ <xs:element name="E">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="A" type="xs:string" dfdl:lengthKind="explicit"
+ dfdl:length="1" minOccurs="0" />
+ <xs:element name="C" type="xs:string" minOccurs="0" dfdl:lengthKind="explicit"
+ dfdl:length="1" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ </tdml:defineSchema>
+
+ <tdml:unparserTestCase name="unseparated_optionals1" root="E" model="s1" roundTrip="none">
+ <tdml:document>1D</tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:E>
+ <A>1</A>
+ <C>D</C>
+ </ex:E>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:unparserTestCase>
+
+</tdml:testSuite>
diff --git a/daffodil-test-ibm1/src/test/scala/org/apache/daffodil/TestNilled.scala b/daffodil-test-ibm1/src/test/scala/org/apache/daffodil/TestNilled.scala
new file mode 100644
index 0000000..8732099
--- /dev/null
+++ b/daffodil-test-ibm1/src/test/scala/org/apache/daffodil/TestNilled.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil
+
+import org.apache.daffodil.tdml.Runner
+import org.junit.Test
+
+object TestNilled {
+ val testDir = "/test-suite/tresys-contributed/"
+ lazy val runner = Runner(testDir, "nilled.tdml")
+}
+
+class TestNilled {
+ import TestNilled._
+
+ @Test def test_nilled_ES_optional() = { runner.trace.runOneTest("nilled_ES_optional") }
+}
diff --git a/daffodil-test-ibm1/src/test/scala/org/apache/daffodil/TestUnseparated.scala b/daffodil-test-ibm1/src/test/scala/org/apache/daffodil/TestUnseparated.scala
new file mode 100644
index 0000000..291591d
--- /dev/null
+++ b/daffodil-test-ibm1/src/test/scala/org/apache/daffodil/TestUnseparated.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil
+
+import org.apache.daffodil.tdml.Runner
+import org.junit.Test
+
+object TestUnseparated {
+ val testDir = "/test-suite/tresys-contributed/"
+ lazy val runner = Runner(testDir, "unseparated.tdml")
+}
+
+class TestUnseparated {
+ import TestUnseparated._
+
+ @Test def test_unseparated_optionals1() = { runner.runOneTest("unseparated_optionals1") }
+
+}