You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by sl...@apache.org on 2020/09/04 17:27:46 UTC
[incubator-daffodil] branch master updated: Stream infoset events
while parsing and reduce memory usage
This is an automated email from the ASF dual-hosted git repository.
slawrence pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git
The following commit(s) were added to refs/heads/master by this push:
new b0f59ef Stream infoset events while parsing and reduce memory usage
b0f59ef is described below
commit b0f59ef7c8b3a1088183c40691df7f3fd10ff864
Author: Steve Lawrence <sl...@apache.org>
AuthorDate: Thu Aug 13 21:15:46 2020 -0400
Stream infoset events while parsing and reduce memory usage
- Create a new InfosetWalker class that walks the infoset with the
ability to pause and continue were it last paused. A new
infosetWalkerBlockCount reference counter is used to determine when
the backtracking is possible and so the walker must pause.
Additionally, isFinal is extended to all types (including simple
types) and set during parsing so that the infoset walker knows when
nodes will have no more changes (e.g. DIComplex/DIArray will have no
more children added)
- Use this new InfosetWalker anytime an infoset is needed (e.g. parse
results, debugging, validation, etc). The visit() methods are no
longer needed and are removed.
- Change the XMLTextInfosetOutputter to output empty complex types as
<foo></foo> rather than <foo />. When we get a startComplex event, we
may not know yet if there are any children, so we can't immediately
create a self closing start tag. Instead, always create an open tag,
and be careful about how we create a close tag so we don't create
empty complex types with newlines in them.
- Remove special handling of root element on DIDocument. Treat it just
like any other DIComplex, except we only care about the zero'th
element. There was a bug in special casing that caused issues with
streaming elements.
- Set DINodes to null after creating end events for them. This allows
the java garbage collector to clean up these elements to allow for
true streaming style behavior when possible. The one case where we
cannot do this is when elements are used in expressions. For now, they
simply are not freed until parse completes.
DAFFODIL-934
---
.../apache/daffodil/debugger/TestCLIDebugger.scala | 4 +-
.../dpath/TestDFDLExpressionEvaluation.scala | 2 +-
.../daffodil/dsom/TestSimpleTypeUnions.scala | 63 ++-
.../org/apache/daffodil/infoset/TestInfoset.scala | 2 +-
.../infoset/TestInfosetCursorFromReader.scala | 2 +-
.../scala/org/apache/daffodil/util/MStack.scala | 6 +-
.../processors/unparsers/ElementUnparser.scala | 2 +-
.../daffodil/debugger/InteractiveDebugger.scala | 65 +--
.../org/apache/daffodil/dpath/DFDLXFunctions.scala | 13 +-
.../org/apache/daffodil/infoset/Infoset.scala | 10 +-
.../org/apache/daffodil/infoset/InfosetImpl.scala | 309 ++++++--------
.../apache/daffodil/infoset/InfosetWalker.scala | 453 +++++++++++++++++++++
.../daffodil/infoset/TeeInfosetOutputter.scala | 86 ++++
.../daffodil/infoset/XMLTextInfosetOutputter.scala | 61 ++-
.../apache/daffodil/processors/DataProcessor.scala | 139 ++++---
.../processors/parsers/ElementKindParsers.scala | 21 +
.../parsers/ExpressionEvaluatingParsers.scala | 1 +
.../parsers/NilEmptyCombinatorParsers.scala | 1 +
.../daffodil/processors/parsers/PState.scala | 68 +++-
.../daffodil/processors/parsers/Parser.scala | 22 +
.../processors/parsers/SequenceParserBases.scala | 122 +++++-
.../parsers/UnseparatedSequenceParsers.scala | 3 +-
22 files changed, 1081 insertions(+), 374 deletions(-)
diff --git a/daffodil-cli/src/it/scala/org/apache/daffodil/debugger/TestCLIDebugger.scala b/daffodil-cli/src/it/scala/org/apache/daffodil/debugger/TestCLIDebugger.scala
index d748b70..394e77b 100644
--- a/daffodil-cli/src/it/scala/org/apache/daffodil/debugger/TestCLIDebugger.scala
+++ b/daffodil-cli/src/it/scala/org/apache/daffodil/debugger/TestCLIDebugger.scala
@@ -1022,7 +1022,7 @@ class TestCLIdebugger {
shell.sendLine("continue")
shell.expect(contains("<a>2</a>"))
- shell.expect(contains("<g />"))
+ shell.expect(contains("<g></g>"))
} finally {
shell.close()
}
@@ -1097,7 +1097,7 @@ class TestCLIdebugger {
shell.expect(contains("hidden: true"))
shell.sendLine("continue")
- shell.expect(contains("<h />"))
+ shell.expect(contains("<h></h>"))
} finally {
shell.close()
}
diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/dpath/TestDFDLExpressionEvaluation.scala b/daffodil-core/src/test/scala/org/apache/daffodil/dpath/TestDFDLExpressionEvaluation.scala
index 7b90a62..bf25f4c 100644
--- a/daffodil-core/src/test/scala/org/apache/daffodil/dpath/TestDFDLExpressionEvaluation.scala
+++ b/daffodil-core/src/test/scala/org/apache/daffodil/dpath/TestDFDLExpressionEvaluation.scala
@@ -50,7 +50,7 @@ class TestDFDLExpressionEvaluation extends Parsers {
val dis = InputSourceDataInputStream(java.nio.ByteBuffer.allocate(0)) // fake. Zero bits available.
val outputter = new NullInfosetOutputter()
- val pstate = PState.createInitialPState(doc, erd, dis, outputter, dp)
+ val pstate = PState.createInitialPState(doc, erd, dis, outputter, dp, areDebugging = false)
val result = compiledExpr.evaluate(pstate)
body(result)
}
diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/dsom/TestSimpleTypeUnions.scala b/daffodil-core/src/test/scala/org/apache/daffodil/dsom/TestSimpleTypeUnions.scala
index 6cc9331..98ae9bf 100644
--- a/daffodil-core/src/test/scala/org/apache/daffodil/dsom/TestSimpleTypeUnions.scala
+++ b/daffodil-core/src/test/scala/org/apache/daffodil/dsom/TestSimpleTypeUnions.scala
@@ -51,7 +51,18 @@ class TestSimpleTypeUnions {
<xs:simpleType name="oneOrTwo">
<xs:union memberTypes="ex:int1Type ex:int2Type"/>
</xs:simpleType>
- <xs:element name="e1" dfdl:lengthKind="delimited" type="ex:oneOrTwo"/>)
+ <xs:element name="e1" dfdl:lengthKind="delimited" type="ex:oneOrTwo">
+ <xs:annotation>
+ <xs:appinfo source="http://www.ogf.org/dfdl/">
+ <!--
+ this assert always passes, but uses e1 in an expression to prevent
+ the InfosetWalker from freeing it, which allows the tests to
+ inspect runtime internals
+ -->
+ <dfdl:assert test="{ fn:true() or /ex:e1 eq 0 }" />
+ </xs:appinfo>
+ </xs:annotation>
+ </xs:element>)
@Test def testUnion01: Unit = {
@@ -86,7 +97,7 @@ class TestSimpleTypeUnions {
@Test def testUnionFirstUnionMemberOk: Unit = {
val (result, actual) = TestUtils.testString(testSchema1, "1")
- val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].root.asInstanceOf[DISimple]
+ val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].contents(0).asInstanceOf[DISimple]
val umstrd = i.unionMemberRuntimeData.get
assertEquals("ex:int1Type", umstrd.diagnosticDebugName)
assertTrue(i.valid.get)
@@ -96,7 +107,7 @@ class TestSimpleTypeUnions {
@Test def testUnionSecondUnionMemberOk: Unit = {
val (result, actual) = TestUtils.testString(testSchema1, "2")
- val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].root.asInstanceOf[DISimple]
+ val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].contents(0).asInstanceOf[DISimple]
val umstrd = i.unionMemberRuntimeData.get
assertEquals("ex:int2Type", umstrd.diagnosticDebugName)
assertTrue(i.valid.get)
@@ -106,7 +117,7 @@ class TestSimpleTypeUnions {
@Test def testUnionNoUnionMemberOK: Unit = {
val (result, _) = TestUtils.testString(testSchema1, "3")
- val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].root.asInstanceOf[DISimple]
+ val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].contents(0).asInstanceOf[DISimple]
val Some(dv: java.lang.Integer) = Some(i.dataValue.getInt)
assertEquals(3, dv.intValue())
assertTrue(i.unionMemberRuntimeData.isEmpty)
@@ -179,11 +190,22 @@ class TestSimpleTypeUnions {
</xs:simpleType>
</xs:union>
</xs:simpleType>
- <xs:element name="e1" dfdl:lengthKind="delimited" type="ex:eType"/>)
+ <xs:element name="e1" dfdl:lengthKind="delimited" type="ex:eType">
+ <xs:annotation>
+ <xs:appinfo source="http://www.ogf.org/dfdl/">
+ <!--
+ this assert always passes, but uses e1 in an expression to prevent
+ the InfosetWalker from freeing it, which allows the tests to
+ inspect runtime internals
+ -->
+ <dfdl:assert test="{ fn:true() or /ex:e1 eq 0 }" />
+ </xs:appinfo>
+ </xs:annotation>
+ </xs:element>)
@Test def testUnionNot3: Unit = {
val (result, _) = TestUtils.testString(testSchema2, "3")
- val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].root.asInstanceOf[DISimple]
+ val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].contents(0).asInstanceOf[DISimple]
val Some(dv: java.lang.Integer) = Some(i.dataValue.getInt)
assertEquals(3, dv.intValue())
assertTrue(i.unionMemberRuntimeData.isEmpty)
@@ -208,7 +230,7 @@ class TestSimpleTypeUnions {
@Test def testUnionNot3_01: Unit = {
val (result, actual) = TestUtils.testString(testSchema2, "1")
- val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].root.asInstanceOf[DISimple]
+ val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].contents(0).asInstanceOf[DISimple]
val umstrd = i.unionMemberRuntimeData.get
assertEquals("ex:int12Type", umstrd.diagnosticDebugName)
assertTrue(i.valid.get)
@@ -218,7 +240,7 @@ class TestSimpleTypeUnions {
@Test def testUnionNot3_02: Unit = {
val (result, actual) = TestUtils.testString(testSchema2, "2")
- val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].root.asInstanceOf[DISimple]
+ val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].contents(0).asInstanceOf[DISimple]
val umstrd = i.unionMemberRuntimeData.get
assertEquals("ex:int12Type", umstrd.diagnosticDebugName)
assertTrue(i.valid.get)
@@ -228,7 +250,7 @@ class TestSimpleTypeUnions {
@Test def testUnionNot3_03: Unit = {
val (result, actual) = TestUtils.testString(testSchema2, "-1")
- val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].root.asInstanceOf[DISimple]
+ val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].contents(0).asInstanceOf[DISimple]
val umstrd = i.unionMemberRuntimeData.get
assertEquals("ex:negIntType", umstrd.diagnosticDebugName) // anonymous simple type gets this name from base.
assertTrue(i.valid.get)
@@ -270,11 +292,22 @@ class TestSimpleTypeUnions {
<xs:pattern value="foo[1234]bar"/>
</xs:restriction>
</xs:simpleType>
- <xs:element name="e1" dfdl:lengthKind="delimited" type="ex:foo3bar"/>)
+ <xs:element name="e1" dfdl:lengthKind="delimited" type="ex:foo3bar">
+ <xs:annotation>
+ <xs:appinfo source="http://www.ogf.org/dfdl/">
+ <!--
+ this assert always passes, but uses e1 in an expression to prevent
+ the InfosetWalker from freeing it, which allows the tests to
+ inspect runtime internals
+ -->
+ <dfdl:assert test="{ fn:true() or /ex:e1 eq '' }" />
+ </xs:appinfo>
+ </xs:annotation>
+ </xs:element>)
@Test def testRestrictionOnUnion_01: Unit = {
val (result, actual) = TestUtils.testString(testSchema3, "foo3bar")
- val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].root.asInstanceOf[DISimple]
+ val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].contents(0).asInstanceOf[DISimple]
val umstrd = i.unionMemberRuntimeData.get
assertEquals("ex:foo3or4bar", umstrd.diagnosticDebugName)
assertTrue(i.valid.get)
@@ -284,7 +317,7 @@ class TestSimpleTypeUnions {
@Test def testRestrictionOnUnion_02: Unit = {
val (result, actual) = TestUtils.testString(testSchema3, "foo1bar")
- val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].root.asInstanceOf[DISimple]
+ val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].contents(0).asInstanceOf[DISimple]
val umstrd = i.unionMemberRuntimeData.get
assertEquals("ex:foo1or2bar", umstrd.diagnosticDebugName)
assertTrue(i.valid.get)
@@ -294,7 +327,7 @@ class TestSimpleTypeUnions {
@Test def testRestrictionOnUnion_03: Unit = {
val (result, actual) = TestUtils.testString(testSchema3, "foo2bar")
- val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].root.asInstanceOf[DISimple]
+ val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].contents(0).asInstanceOf[DISimple]
val umstrd = i.unionMemberRuntimeData.get
assertEquals("ex:foo1or2bar", umstrd.diagnosticDebugName)
assertTrue(i.valid.get)
@@ -304,7 +337,7 @@ class TestSimpleTypeUnions {
@Test def testRestrictionOnUnionFail_01: Unit = {
val (result, _) = TestUtils.testString(testSchema3, "foo4bar")
- val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].root.asInstanceOf[DISimple]
+ val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].contents(0).asInstanceOf[DISimple]
val Some(dv: String) = Some(i.dataValue.getString)
assertEquals("foo4bar", dv)
assertTrue(i.unionMemberRuntimeData.isEmpty)
@@ -333,7 +366,7 @@ class TestSimpleTypeUnions {
*/
@Test def testRestrictionOnUnionFail_02: Unit = {
val (result, _) = TestUtils.testString(testSchema3, "notfoo1bar")
- val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].root.asInstanceOf[DISimple]
+ val i = result.resultState.asInstanceOf[PState].infoset.asInstanceOf[DIDocument].contents(0).asInstanceOf[DISimple]
val Some(dv: String) = Some(i.dataValue.getString)
assertEquals("notfoo1bar", dv)
assertTrue(i.unionMemberRuntimeData.isEmpty)
diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/infoset/TestInfoset.scala b/daffodil-core/src/test/scala/org/apache/daffodil/infoset/TestInfoset.scala
index 26ffbf0..901a508 100644
--- a/daffodil-core/src/test/scala/org/apache/daffodil/infoset/TestInfoset.scala
+++ b/daffodil-core/src/test/scala/org/apache/daffodil/infoset/TestInfoset.scala
@@ -66,7 +66,7 @@ object TestInfoset {
val infosetRootNode = {
val ustate = unparseResult.resultState.asInstanceOf[UStateMain]
val diDocument: DIDocument = ustate.documentElement
- val rootElement = diDocument.getRootElement().asInstanceOf[DIElement]
+ val rootElement = diDocument.contents(0).asInstanceOf[DIElement]
Assert.invariant(rootElement ne null)
rootElement
}
diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/infoset/TestInfosetCursorFromReader.scala b/daffodil-core/src/test/scala/org/apache/daffodil/infoset/TestInfosetCursorFromReader.scala
index d2f1bd3..8c4aa0b 100644
--- a/daffodil-core/src/test/scala/org/apache/daffodil/infoset/TestInfosetCursorFromReader.scala
+++ b/daffodil-core/src/test/scala/org/apache/daffodil/infoset/TestInfosetCursorFromReader.scala
@@ -278,7 +278,7 @@ class TestInfosetInputterFromReader {
val Seq(fooERD: ElementRuntimeData) = barSeqTRD.groupMembers
val doc = inp.documentElement
val Start(bar_s: DIComplex) = is.next
- doc.addChild(bar_s)
+ doc.addChild(bar_s, tunable)
inp.pushTRD(barSeqTRD)
inp.pushTRD(fooERD)
val StartArray(foo_arr_s) = is.next
diff --git a/daffodil-lib/src/main/scala/org/apache/daffodil/util/MStack.scala b/daffodil-lib/src/main/scala/org/apache/daffodil/util/MStack.scala
index 9b6984d..a0899da 100644
--- a/daffodil-lib/src/main/scala/org/apache/daffodil/util/MStack.scala
+++ b/daffodil-lib/src/main/scala/org/apache/daffodil/util/MStack.scala
@@ -269,9 +269,9 @@ protected abstract class MStack[@specialized T] private[util] (
*
* @return the element on top of the stack.
*/
- @inline final def top: T = table(index - 1).asInstanceOf[T]
+ @inline final def top: T = table(index - 1)
- @inline final def bottom: T = table(0).asInstanceOf[T]
+ @inline final def bottom: T = table(0)
@inline final def isEmpty: Boolean = index == 0
@@ -307,7 +307,7 @@ protected abstract class MStack[@specialized T] private[util] (
Assert.usage(hasNext)
Assert.usage(initialIndex <= index) // can't make it smaller than when initialized.
currentIndex -= 1
- table(currentIndex).asInstanceOf[T]
+ table(currentIndex)
}
/**
diff --git a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/ElementUnparser.scala b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/ElementUnparser.scala
index f70f76d..9c46b82 100644
--- a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/ElementUnparser.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/ElementUnparser.scala
@@ -431,7 +431,7 @@ sealed trait RegularElementUnparserStartEndStrategy
c.addChild(res, state.tunable)
} else {
val doc = state.documentElement
- doc.addChild(res) // DIDocument, which is never a current node, must have the child added
+ doc.addChild(res, state.tunable) // DIDocument, which is never a current node, must have the child added
doc.setFinal() // that's the only child.
}
res
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/debugger/InteractiveDebugger.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/debugger/InteractiveDebugger.scala
index bc41fb5..594c5a4 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/debugger/InteractiveDebugger.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/debugger/InteractiveDebugger.scala
@@ -450,12 +450,22 @@ class InteractiveDebugger(runner: InteractiveDebuggerRunner, eCompilers: Express
}
}
- private def debugPrettyPrintXML(ie: InfosetElement): Unit = {
+ private def infosetToString(ie: InfosetElement): String = {
val bos = new java.io.ByteArrayOutputStream()
val xml = new XMLTextInfosetOutputter(bos, true)
- ie.visit(xml, DebuggerConfig.removeHidden)
- xml.endDocument() // causes the outputter to flush to the stream
- debugPrintln(bos.toString("UTF-8"))
+ val iw = InfosetWalker(
+ ie.asInstanceOf[DINode],
+ xml,
+ walkHidden = !DebuggerConfig.removeHidden,
+ ignoreBlocks = true,
+ removeUnneeded = false)
+ iw.walk()
+ bos.toString("UTF-8")
+ }
+
+ private def debugPrettyPrintXML(ie: InfosetElement): Unit = {
+ val infosetString = infosetToString(ie)
+ debugPrintln(infosetString)
}
/**********************************/
@@ -1499,38 +1509,35 @@ class InteractiveDebugger(runner: InteractiveDebuggerRunner, eCompilers: Express
def act(args: Seq[String], prestate: StateForDebugger, state: ParseOrUnparseState, processor: Processor): DebugState.Type = {
debugPrintln("%s:".format(name))
- var infoset: DIElement = null
if (state.hasInfoset) {
- infoset = state.infoset
- if (DebuggerConfig.infosetParents < 0) {
- infoset = infoset.toRootDoc.getRootElement().asInstanceOf[DIElement]
- } else {
- (1 to DebuggerConfig.infosetParents).foreach { n =>
- if (infoset.diParent != null) {
- infoset = infoset.diParent
- }
- }
+ var parentSteps =
+ if (DebuggerConfig.infosetParents < 0) Int.MaxValue
+ else DebuggerConfig.infosetParents
+
+ var node = state.infoset
+ while (parentSteps > 0 && node.diParent != null) {
+ node = node.diParent
+ parentSteps -= 1
}
- }
- if (infoset != null) {
- val bos = new java.io.ByteArrayOutputStream()
- val xml = new XMLTextInfosetOutputter(bos, true)
- infoset.visit(xml, DebuggerConfig.removeHidden)
- xml.endDocument() // causes the outputter to flush to the stream
- val infosetString = bos.toString("UTF-8")
- val lines = infosetString.split("\n")
- val tailLines =
- if (DebuggerConfig.infosetLines > 0) {
- val dropCount = lines.size - DebuggerConfig.infosetLines
+ node match {
+ case d: DIDocument if d.contents.size == 0 => {
+ debugPrintln("No Infoset", " ")
+ }
+ case _ => {
+ val infosetString = infosetToString(node)
+ val lines = infosetString.split("\n")
+
+ val dropCount =
+ if (DebuggerConfig.infosetLines < 0) 0
+ else Math.max(0, lines.size - DebuggerConfig.infosetLines)
if (dropCount > 0) {
debugPrintln("...", " ")
}
- lines.drop(dropCount)
- } else {
- lines
+ val linesToShow = lines.drop(dropCount)
+ linesToShow.foreach { l => debugPrintln(l, " ") }
}
- tailLines.foreach(l => debugPrintln(l, " "))
+ }
} else {
debugPrintln("No Infoset", " ")
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/dpath/DFDLXFunctions.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/dpath/DFDLXFunctions.scala
index aa1add8..85cc6b4 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/dpath/DFDLXFunctions.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/dpath/DFDLXFunctions.scala
@@ -20,10 +20,9 @@ package org.apache.daffodil.dpath
import java.math.{ BigInteger => JBigInt }
import org.apache.daffodil.exceptions.Assert
+import org.apache.daffodil.infoset.DINode
import org.apache.daffodil.infoset.DISimple
import org.apache.daffodil.infoset.DataValue.DataValuePrimitive
-import org.apache.daffodil.infoset.InfosetCommon
-import org.apache.daffodil.infoset.XMLTextInfosetOutputter
import org.apache.daffodil.processors.parsers.PState
import org.apache.daffodil.processors.parsers.ParseError
import org.apache.daffodil.processors.unparsers.UState
@@ -34,14 +33,6 @@ import org.apache.daffodil.util.Maybe.One
case class DFDLXTrace(recipe: CompiledDPath, msg: String)
extends RecipeOpWithSubRecipes(recipe) {
- private def asXMLString(ie: InfosetCommon) = {
- val bos = new java.io.ByteArrayOutputStream()
- val xml = new XMLTextInfosetOutputter(bos, true)
- ie.visit(xml, false)
- xml.endDocument() // causes the outputter to flush to the stream
- bos.toString("UTF-8")
- }
-
override def run(dstate: DState): Unit = {
recipe.run(dstate)
val nodeString: String = dstate.currentNode match {
@@ -52,7 +43,7 @@ case class DFDLXTrace(recipe: CompiledDPath, msg: String)
dstate.setCurrentValue(v)
v.toString()
}
- case other: InfosetCommon => "\n" + asXMLString(other)
+ case other: DINode => other.namedQName.toString
}
System.err.println("trace " + msg + ":" + nodeString)
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/Infoset.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/Infoset.scala
index e07aba7..249dcc7 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/Infoset.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/Infoset.scala
@@ -27,11 +27,7 @@ import org.apache.daffodil.xml.NS
object INoWarn2 { ImplicitsSuppressUnusedImportWarning() }
-trait InfosetCommon {
- def visit(handler: InfosetOutputter, removeHidden: Boolean = true): Unit
-}
-
-trait InfosetArray extends InfosetCommon {
+trait InfosetArray {
def append(ie: InfosetElement): Unit
def getOccurrence(occursIndex: Long): InfosetElement
def length: Long
@@ -94,11 +90,9 @@ trait InfosetSimpleElement extends InfosetElement {
}
trait InfosetDocument extends InfosetItem {
- def getRootElement(): InfosetElement
- def setRootElement(root: InfosetElement, tunable: DaffodilTunables): Unit
}
-trait InfosetItem extends InfosetCommon {
+trait InfosetItem {
/**
* The totalElementCount is the total count of how many elements this InfosetItem contains.
*
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetImpl.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetImpl.scala
index f8503f5..648bcc8 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetImpl.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetImpl.scala
@@ -65,7 +65,7 @@ import org.apache.daffodil.xml.XMLUtils
import passera.unsigned.ULong
import org.apache.daffodil.dsom.DPathCompileInfo
-sealed trait DINode {
+sealed trait DINode extends DIFinalizable {
def asSimple: DISimple = {
this match {
@@ -92,12 +92,15 @@ sealed trait DINode {
else asComplex
def isDefaulted: Boolean
+ def isHidden: Boolean
def children: Stream[DINode]
def totalElementCount: Long
def namedQName: NamedQName
def erd: ElementRuntimeData
+ def infosetWalkerBlockCount: Int
+
/**
* Can treat any DINode, even simple ones, as a container of other nodes.
* This simplifies walking an infoset.
@@ -105,8 +108,27 @@ sealed trait DINode {
def contents: IndexedSeq[DINode]
final def numChildren = contents.length
- def visit(handler: InfosetOutputter, removeHidden: Boolean = true): Unit
+ /**
+ * Returns the DINode that contains this DINode. Note that this behavior is
+ * slightly different from the "parent". The parent method always returns a
+ * DIComplex, even if this DINode is in an array. This usually makes sense
+ * and is what the caller intended when they call parent.
+ *
+ * However, we sometimes want the exact DINode that contains this DINode. So
+ * if this DINode is a DIElement in an array, then this returns the DIArray.
+ * Otherwise (i.e. DIArray or non-array DIElement) then it should return the
+ * DIComplex parent. Note that if the node is a DIDocument it should return
+ * null, which mimics the behavior of .parent.
+ */
+ def containerNode: DINode
+ /**
+ * Free memory associated with the child at a specified index. Note that this
+ * should only free children if they are not actually needed anymore. It can
+ * assume that the infoset events have been created, but if, for example, a
+ * child is used in DPath expressions this should not free the child.
+ */
+ def freeChildIfNoLongerNeeded(index: Int): Unit
}
/**
@@ -813,6 +835,17 @@ sealed trait DIElement
override final def trd = erd
/**
+ * Used to prevent the infoset walker from walking into an infoset element.
+ * This is generally incremented when marks are added and decremented when
+ * they are discarded. The idea being that we do not want to walk into
+ * infoset elements if there could be backtracking. Though this is not
+ * strictly the only time blocks may be added. For example, we also increment
+ * this when in unordered sequences to prevent walking until elements are
+ * reordered to schema definition order.
+ */
+ var infosetWalkerBlockCount: Int = 0
+
+ /**
* Tells us if the element was entirely created by defaulting elements.
*/
def isDefaulted: Boolean
@@ -878,6 +911,10 @@ sealed trait DIElement
override def parent = _parent
+ override def containerNode =
+ if (array.isDefined) array.get.asInstanceOf[DINode]
+ else parent.asInstanceOf[DINode]
+
def diParent = _parent.asInstanceOf[DIComplex]
override def setParent(p: InfosetComplexElement): Unit = {
@@ -943,11 +980,12 @@ final class DIArray(
// without bound. So, replace this with a mutable map so that it can shrink
// as well as grow. // not saved. Needed only to get initial size.
)
- extends DINode with InfosetArray
- with DIFinalizable {
+ extends DINode with InfosetArray {
private lazy val nfe = new InfosetArrayNotFinalException(this)
+ override def containerNode = parent
+
override def requireFinal: Unit = {
if (!isFinal) {
// If this DIArray isn't final, either we haven't gotten all of its
@@ -971,6 +1009,38 @@ final class DIArray(
override def isSimple = false
override def isComplex = false
+ // Parsers don't actually call setHidden on DIArrays, only on DIElements. But
+ // DIArrays are always created when there is at least one child element, and
+ // since all children of an array must have the same visibility, we can just
+ // inspect the isHidden property of the first child to determine if the
+ // entire array is hidden.
+ override def isHidden = _contents(0).isHidden
+
+ // Marks/blocks are never taken on arrays, so the infosetWalkerBlockCount
+ // would normally be zero for arrays. The infosetWalkerBlockCount is used to determine
+ // if this array is known to exist or if it could be backtracked. We know
+ // this DIArray exists if the first child is known to exist (i.e. if it has
+ // no blocks).
+ //
+ // Additionally, the infoset walker may delete elements from an array. In
+ // which case _contents(0) may be null. But if _contents(0) is null, that
+ // means there was an element that was known to exist, and it was freed after
+ // its event were created. So that doesn't cause a block.
+ //
+ // We also add the parent block count. This is because a mark is taken when
+ // speculatively parsing a single element of an array. This mark increments
+ // the infosetWalkerBlockCount of the /parent/ of the DIArray because the new
+ // DIElement in this DIArray does not yet exist. So as long as we are
+ // speculatively parsing a single element of an array, this block on the
+ // parent prevents walking into this new speculatively parsed array element.
+ // Once we finish speculatively parsing that single array element, that block
+ // is removed, and we immediately call walk() to walk the new array element
+ // (assuming there are no other PoU's in scope). This process is repeated for
+ // each speculatively parsed array element.
+ override def infosetWalkerBlockCount =
+ parent.infosetWalkerBlockCount +
+ (if (_contents(0) == null) 0 else _contents(0).infosetWalkerBlockCount)
+
override def toString = "DIArray(" + namedQName + "," + _contents + ")"
def namedQName = erd.namedQName
@@ -1030,19 +1100,15 @@ final class DIArray(
a
}
- final def visit(handler: InfosetOutputter, removeHidden: Boolean = true): Unit = {
- // Do not create an event if there's nothing in the array or if the array
- // is hidden. Unfortunately, there is no way to tell if an array is hidden,
- // only its elements. But if the first element is hidden, they are all
- // hidden, so we check that to determine if the array is hidden.
- if (length > 0 && (!_contents(0).isHidden || !removeHidden)) {
- handler.startArray(this)
- _contents.foreach { _.visit(handler, removeHidden) }
- handler.endArray(this)
+ final def isDefaulted: Boolean = children.forall { _.isDefaulted }
+
+ final def freeChildIfNoLongerNeeded(index: Int): Unit = {
+ val node = _contents(index)
+ if (!node.erd.dpathElementCompileInfo.isReferencedByExpressions) {
+ // set to null so that the garbage collector can free this node
+ _contents(index) = null
}
}
-
- final def isDefaulted: Boolean = children.forall { _.isDefaulted }
}
/**
@@ -1296,16 +1362,23 @@ sealed class DISimple(override val erd: ElementRuntimeData)
override def totalElementCount = 1L
- final def visit(handler: InfosetOutputter, removeHidden: Boolean = true): Unit = {
- if (!this.isHidden || !removeHidden) {
- handler.startSimple(this)
- handler.endSimple(this)
- }
+ /**
+ * requireFinal is only ever used on unparse, and we never need to require a
+ * simple type to be final during unparse. However, we do need to have an
+ * implementation of this function, as DISimple needs DIFinalizable and
+ * isFinal for parsing and allowing the cleanup of unneeded DINodes.
+ */
+ override def requireFinal: Unit = {
+ Assert.invariantFailed("Should not requireFinal a simple type")
+ }
+
+ final def freeChildIfNoLongerNeeded(index: Int): Unit = {
+ Assert.invariantFailed("Should not try to remove a child of a simple type")
}
}
/**
- * Arrays and complex elements have a notion of being finalized, when unparsing.
+ * When unparsing, arrays and complex elements have a notion of being finalized.
* This is when we know that no more elements will be added to them, so things
* like fn:count can return a value knowing it won't change, and fn:exists can
* return false, knowing nobody will subsequently append the item that was being
@@ -1325,31 +1398,19 @@ sealed trait DIFinalizable {
}
/**
- * Complex elements have an array of slots one per named child element.
- *
- * TODO: consider xs:choice - alternatives could share slots, but that would
- * add a lot of complexity, and the nil technique of storing null in a
- * slot to indicate a nilled element only works if we have a positive association
- * of slots to element-bases. If we were to share slots we'd need a different way
- * to indicate nil. A better approach for xs:choice would be a sparse table of
- * slots (whenever there are more than N - some threshold), so that we're not
- * allocating arrays of 200 slots just because there are 200 branches of a choice.
- *
- * A slot stores a Maybe[InfosetCommonMixin]. None means not present (yet, because it
- * hasn't been parsed yet, or it is an optional element (minOccurs 0, maxOccurs 1)
- * and is not present.) One[DISimple] or One[DIComplex] mean a required element
- * is present, or an optional element (minOccurs 0, maxOccurs 1) is present.
- *
- * A slot of a DIComplex should never be null.
- *
- * One[DIArray] means the slot is for a recurring element which can have 2+ instances.
- * The DIArray object's length gives the number of occurrences.
+ * Complex elements have an array of DINodes. Nodes may either be DISimple for
+ * simple types, DIComplex for complex types, or DIArray for arrays or optional
+ * elements. Note that even if an element is nilled, there is still a DIElement
+ * (i.e. a null child element does not mean the child is nilled). Also note
+ * that in some cases Daffodil will remove children if they are determined they
+ * are no longer needed to free up memory. It does this by setting the child to
+ * null. So child slots can have a null value, but only in cases where Daffodil
+ * should never need access to that child.
*/
sealed class DIComplex(override val erd: ElementRuntimeData)
extends DIElement
with DIComplexSharedImplMixin
with InfosetComplexElement
- with DIFinalizable // with HasModelGroupMixin
{ diComplex =>
final override def isSimple = false
@@ -1368,10 +1429,6 @@ sealed class DIComplex(override val erd: ElementRuntimeData)
private lazy val nfe = new InfosetComplexElementNotFinalException(this)
- override def requireFinal: Unit = {
- if (!isFinal) throw nfe
- }
-
override def valueStringForDebug: String = ""
final override def isEmpty: Boolean = false
@@ -1388,6 +1445,10 @@ sealed class DIComplex(override val erd: ElementRuntimeData)
}
}
+ override def requireFinal: Unit = {
+ if (!isFinal) throw nfe
+ }
+
val childNodes = new ArrayBuffer[DINode]
//
// TODO: Cleanup - Change below to use NonAllocatingMap to improve code style.
@@ -1441,6 +1502,14 @@ sealed class DIComplex(override val erd: ElementRuntimeData)
}
}
+ def freeChildIfNoLongerNeeded(index: Int): Unit = {
+ val node = childNodes(index)
+ if (!node.erd.dpathElementCompileInfo.isReferencedByExpressions) {
+ // set to null so that the garbage collector can free this node
+ childNodes(index) = null
+ }
+ }
+
/*
* When parsing unordered sequences it is possible to have non-contiguous arrays.
* When parsed, these arrays show up as separate DIArrays in the infoset. We need
@@ -1495,9 +1564,13 @@ sealed class DIComplex(override val erd: ElementRuntimeData)
if (!e.isHidden && !hasVisibleChildren) hasVisibleChildren = true
if (e.runtimeData.isArray) {
val childERD = e.runtimeData
- if (childNodes.isEmpty || childNodes.last.erd != childERD) {
- // no children, or last child is not a DIArray for
- // this element, create the DIArray
+ if (childNodes.lastOption.map { n => n == null || n.erd != childERD }.getOrElse(true)) {
+ // This complex element either has no children, the most recently added
+ // child is different than the new child, or the last child is null
+ // (which means it was freed because it was finished, and thus must be
+ // different from this new element). This element is an array, so in
+ // any of these cases, we must create a new DIArray, and then we'll add
+ // the new element to it.
val ia = new DIArray(childERD, this, tunable.initialElementOccurrencesHint.toInt)
addChildToFastLookup(ia)
childNodes += ia
@@ -1621,13 +1694,6 @@ sealed class DIComplex(override val erd: ElementRuntimeData)
a
}
- override def visit(handler: InfosetOutputter, removeHidden: Boolean = true): Unit = {
- if (!this.isHidden || !removeHidden) {
- handler.startComplex(this)
- childNodes.foreach(node => node.visit(handler, removeHidden))
- handler.endComplex(this)
- }
- }
}
/*
@@ -1638,7 +1704,6 @@ final class DIDocument(
erd: ElementRuntimeData)
extends DIComplex(erd)
with InfosetDocument {
- var root: DIElement = null
/**
* Set if this DIDocument is being attached to a DIElement just to establish the
@@ -1646,44 +1711,6 @@ final class DIDocument(
* a constant value
*/
var isCompileExprFalseRoot: Boolean = false
-
- def setRootElement(rootElement: InfosetElement, tunable: DaffodilTunables): Unit = {
- root = rootElement.asInstanceOf[DIElement]
- addChild(root, tunable)
- }
-
- override def addChild(child: InfosetElement, tunable: DaffodilTunables) =
- addChild(child)
-
- def addChild(child: InfosetElement): Unit = {
- /*
- * DIDocument normally only has a single child.
- * However, if said child wants to create a quasi-element (eg. if it is a simpleType with a typeCalc),
- * said element will need to be temporarily added as a second child to DIDocument.
- * When this happens, we do not set it as the root, but otherwise proceed normally
- */
- val node = child.asInstanceOf[DINode]
- childNodes += node
- if (node.erd.dpathElementCompileInfo.isReferencedByExpressions) {
- val ab = new ArrayBuffer[DINode]()
- ab += node
- nameToChildNodeLookup.put(node.namedQName, ab)
- }
- child.setParent(this)
- if (root == null)
- root = child.asInstanceOf[DIElement]
- }
-
- def getRootElement(): InfosetElement = {
- root
- }
-
- override def visit(handler: InfosetOutputter, removeHidden: Boolean = true): Unit = {
- assert(root != null)
- handler.startDocument()
- root.visit(handler, removeHidden)
- handler.endDocument()
- }
}
object Infoset {
@@ -1697,98 +1724,4 @@ object Infoset {
new DIDocument(erd)
}
- def newDocument(root: InfosetElement, tunable: DaffodilTunables): InfosetDocument = {
- val doc = newDocument(root.runtimeData)
- doc.setRootElement(root, tunable)
- doc
- }
-
- /**
- * Used to convert default value strings (from the XSD default property)
- * into the appropriate representation type.
- *
- * For use during compilation of a schema. Not for runtime, as this
- * can be slow.
- */
- // TODO: consolidate into the NodeInfo object where there is already similar
- // code. Or maybe consolidates into the DPath Conversions.scala code?
- // def convertToInfosetRepType(primType: PrimType, value: String, context: ThrowsSDE): AnyRef = {
- // import NodeInfo.PrimType._
- // val res = primType match {
- // case String => value
- // case Int => value.toInt
- // case Byte => value.toByte
- // case Short => value.toShort
- // case Long => value.toLong
- // case Integer => new java.math.BigInteger(value)
- // case Decimal => new java.math.BigDecimal(value)
- // case UnsignedInt => {
- // val res = java.lang.Long.parseLong(value)
- // context.schemaDefinitionUnless(res >= 0, "Cannot convert %s to %s.", value, primType.name)
- // res
- // }
- // case UnsignedByte => {
- // val res = value.toShort
- // context.schemaDefinitionUnless(res >= 0, "Cannot convert %s to %s.", value, primType.name)
- // res
- // }
- // case UnsignedShort => {
- // val res = value.toInt
- // context.schemaDefinitionUnless(res >= 0, "Cannot convert %s to %s.", value, primType.name)
- // res
- // }
- // case UnsignedLong => {
- // val res = new java.math.BigInteger(value)
- // context.schemaDefinitionUnless(res.doubleValue >= 0, "Cannot convert %s to %s.", value, primType.name)
- // res
- // }
- // case NonNegativeInteger => {
- // val res = new java.math.BigInteger(value)
- // context.schemaDefinitionUnless(res.doubleValue >= 0, "Cannot convert %s to %s.", value, primType.name)
- // res
- // }
- // case Double => {
- // value match {
- // case XMLUtils.PositiveInfinityString => scala.Double.PositiveInfinity
- // case XMLUtils.NegativeInfinityString => scala.Double.NegativeInfinity
- // case XMLUtils.NaNString => scala.Double.NaN
- // case _ => value.toDouble
- // }
- // }
- // case Float => {
- // value match {
- // case XMLUtils.PositiveInfinityString => scala.Float.PositiveInfinity
- // case XMLUtils.NegativeInfinityString => scala.Float.NegativeInfinity
- // case XMLUtils.NaNString => scala.Float.NaN
- // case _ => value.toFloat
- // }
- // }
- // case HexBinary => Misc.hex2Bytes(value) // convert hex constant into byte array
- // case Boolean => {
- // if (value == "true") true
- // else if (value == "false") false
- // else context.schemaDefinitionError("Cannot convert %s to %s.", value, primType.name)
- // }
- // case Time => {
- // val cal = new GregorianCalendar()
- // val pos = new java.text.ParsePosition(0)
- // new com.ibm.icu.text.SimpleDateFormat("HH:mm:ssZZZZZ").parse(value, cal, pos)
- // DFDLTime(cal, true)
- // }
- // case DateTime => {
- // val cal = new GregorianCalendar()
- // val pos = new java.text.ParsePosition(0)
- // new com.ibm.icu.text.SimpleDateFormat("uuuu-MM-dd'T'HH:mm:ssZZZZZ").parse(value, cal, pos)
- // DFDLDateTime(cal, true)
- // }
- // case Date => {
- // val cal = new GregorianCalendar()
- // val pos = new java.text.ParsePosition(0)
- // new com.ibm.icu.text.SimpleDateFormat("uuuu-MM-dd").parse(value, cal, pos)
- // DFDLDate(cal, true)
- // }
- // }
- // res.asInstanceOf[AnyRef]
- // }
-
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetWalker.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetWalker.scala
new file mode 100644
index 0000000..bac3327
--- /dev/null
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetWalker.scala
@@ -0,0 +1,453 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.infoset
+
+import org.apache.daffodil.exceptions.Assert
+import org.apache.daffodil.util.MStackOfInt
+import org.apache.daffodil.util.Maybe
+import org.apache.daffodil.util.Maybe.One
+import org.apache.daffodil.util.Maybe.Nope
+
+object InfosetWalker {
+
+ /**
+ * Create an infoset walker starting with a specified DINode. If the caller
+ * starts the InfosetWalker at a specified DINode that is not a DIDocument
+ * (which is often the case with the debugger) the InfosetWalker will walk
+ * the node and its children, but will not walk any siblings of the node.
+ * Regardless whether the DINode is a DIDocument or something else, the
+ * InfosetWalker will still always call the start/endDocument functions of
+ * the InfosetOutputter.
+ *
+ * @param root
+ *
+ * The DINode to start walking the infoset
+ *
+ * @param outputter
+ *
+ * The InfosetOutputter to output events while walking the infoset
+ *
+ * @param walkHidden
+ *
+ * Whether or not to walk infoset elements that are considered hidden. This
+ * should usually only be set to true while debugging
+ *
+ * @param ignoreBlocks
+ *
+ * Whether or not to ignore blocks when walking the infoset, which are used
+ * to prevent creation of infoset events that might be backtracked. This
+ * should usually only be set to true while debugging
+ *
+ * @param removeUnneeded
+ *
+ * Whether or not to remove infoset nodes once it is determined that they
+ * will no longer be used by Daffodil. This should usually be set to true
+ * except while debugging
+ */
+ def apply(
+ root: DINode,
+ outputter: InfosetOutputter,
+ walkHidden: Boolean,
+ ignoreBlocks: Boolean,
+ removeUnneeded: Boolean): InfosetWalker = {
+
+ // Determine the container of the root node and the index in which it
+ // appears in that node
+ val (startingContainerNode, startingContainerIndex) = root match {
+ case d: DIDocument => {
+ // We want to start at the zero'th child index of the document
+ (d, 0)
+ }
+ case _ => {
+ // This case should only be hit when using the debugger to start
+ // walking at a node that isn't the root DIDocument. This gets the
+ // container of the root node to start at and finds the index in that
+ // container
+ (root.containerNode, root.containerNode.contents.indexOf(root))
+ }
+ }
+ new InfosetWalker(
+ startingContainerNode,
+ startingContainerIndex,
+ outputter,
+ walkHidden,
+ ignoreBlocks,
+ removeUnneeded)
+ }
+
+}
+
+/**
+ * The purpose of this class is to walk Daffodil's internal infoset
+ * representation (i.e. DINodes), starting at a specified DINode, and call the
+ * appropriate functions on the InfosetOutputter. The InfosetOutputter provided
+ * to this class determines how the internal infoset is projectted to the user
+ * (e.g. XML, json, SAX events).
+ *
+ * Calling the walk() method causes the InfosetWalker to begin this process. At
+ * any point as determined by the walker, it may pause the walk. Thus, the
+ * entire infoset is not guaranteed to have been walked when the walk() method
+ * returns. In fact, it is possible that no progress could be made. It is up to
+ * the caller to call walk() at appropriate times where it is likely that a
+ * walk could make progress.
+ *
+ * The isFinished() method can be used to determine if the walker walked the
+ * entire infoset or not.
+ *
+ * @param startingContainerNode
+ *
+ * The container DINode of the element to start at. This should be either a
+ * DIComplex, DIDocument, or DIArray. DISimple is technically allowed, but
+ * will not create any useful events as it contains no children.
+ *
+ * @param startingContainerIndex
+ *
+ * The child index of the element in the startingContainerNode
+ *
+ * @param outputter
+ *
+ * The InfosetOutputter to output events while walking the infoset
+ *
+ * @param walkHidden
+ *
+ * Whether or not to walk infoset elements that are considered hidden. This
+ * should usually only be set to true while debugging
+ *
+ * @param ignoreBlocks
+ *
+ * Whether or not to ignore blocks when walking the infoset, which are used
+ * to prevent creation of infoset events that might be backtracked. This
+ * should usually only be set to true while debugging
+ *
+ * @param removeUnneeded
+ *
+ * Whether or not to remove infoset nodes once it is determined that they
+ * will no longer be used by Daffodil. This should usually be set to true
+ * except while debugging
+ */
+class InfosetWalker private (
+ startingContainerNode: DINode,
+ startingContainerIndex: Int,
+ val outputter: InfosetOutputter,
+ walkHidden: Boolean,
+ ignoreBlocks: Boolean,
+ removeUnneeded: Boolean) {
+
+ /**
+ * These two pieces of mutable state are all that is needed to keep track of
+ * where we are in the infoset. The element that the walker will output an
+ * event for when step() is called is referenced by its container DINode
+ * (either a DIComplex/DIDocument or DIArray) which is stored in
+ * containerNode, and its index within that containing node (which is the
+ * value on the top of the containerIndexStack). Once step() creates the
+ * appropriate event for the element, it will mutate this state so the next
+ * call to step creates events for the next element in the infoset.
+ *
+ * To step to the next sibling of an element, we only need to increment the
+ * index on the top of the stack, since siblings have the same container.
+ *
+ * To step into a DIComplex/DIArray, we mutate containerNode to be the
+ * DIComplex/DIArray we want to step into and push a zero onto the container
+ * index stack so that the next element is the zero'th child of that
+ * container.
+ *
+ * To step out of a DIComplex/DIArray, we mutate containerNode to be the
+ * container of the containerNode and pop off the top of the stack. We then
+ * can perform the logic to step to the next sibling.
+ *
+ * Special helper functions are created to make the above logic more clear.
+ *
+ * Note that we initialize the top of the container index stack with one less
+ * than the starting container index. This lets the getNextStep function know
+ * that we have not yet started the document. Once the document is started,
+ * we increment this value on the top of the stack so that the starting index
+ * is correct.
+ */
+ private var containerNode: DINode = startingContainerNode
+ private var containerIndexStack: MStackOfInt = {
+ val stack = MStackOfInt()
+ stack.push(startingContainerIndex - 1)
+ stack
+ }
+
+ private var finished = false
+
+ /**
+ * Determine if the walker has finished walking.
+ */
+ def isFinished = finished
+
+ /**
+ * Take zero or more steps in the infoset. This may or may not walk the
+ * entire infoset. If isFinished returns false, walk can be called again to
+ * continue attempting to walk the infoset where it left off. Because this is
+ * not guaranteed to make any progress, the caller should attempt to only
+ * call walk() when infoset state has changed such that progress may be
+ * possible.
+ *
+ * It is an error to call walk() if isFinished returns true
+ */
+ def walk(): Unit = {
+ Assert.usage(!finished)
+
+ var maybeNextStep: Maybe[InfosetWalkerStep] = Nope
+ while ({
+ maybeNextStep = getNextStep()
+ maybeNextStep.isDefined
+ }) {
+ maybeNextStep.get.step()
+ }
+ }
+
+ /**
+ * Determine the next step to take, if any
+ */
+ private def getNextStep(): Maybe[InfosetWalkerStep] = {
+ if (finished) {
+ Nope
+ } else if ((containerNode ne startingContainerNode) || containerIndexStack.top == startingContainerIndex) {
+ // The containerNode is either some child of the starting node (container
+ // node != starting container code) or we are exactly on the starting
+ // node (container node == starting container node && top of index stack
+ // == starting index). So we can potentially take a normal step.
+ //
+ // This doesn't necessarily mean we have a step though, since there may
+ // be PoU's that could backtrack and so we can't create events yet, or
+ // other kinds of blocks could exist. So we need to inspect the infoset
+ // state to determine if we can actually create events for the current
+ // infoset node and take a step.
+ if (ignoreBlocks || canTakeStep()) {
+ One(InfosetWalkerStepMove)
+ } else {
+ Nope
+ }
+ } else {
+ // We only get here if the container node is the starting container node,
+ // but the top of the index is not the index of the root node. This means
+ // the next step is either a start or end step.
+ //
+ // If the top of the index stack is less than the starting index, then we
+ // have not started the document yet (because we initialize the index
+ // stack with one less than the starting index). The next step must be a
+ // start step. The InfosetWalkerStepStart is responsible for creating the
+ // right event and updating the containerIndexStack to reference the
+ // correct starting index.
+ //
+ // Otherwise the top of the index stack must be greater than the starting
+ // index because we have moved passed the starting element index, and
+ // thus the next step is an end step. The InfosetWalkerStepEnd is
+ // responsible for creating the endDocument event and cleaning up state.
+ if (containerIndexStack.top < startingContainerIndex) {
+ One(InfosetWalkerStepStart)
+ } else {
+ One(InfosetWalkerStepEnd)
+ }
+ }
+ }
+
+ private def canTakeStep(): Boolean = {
+
+ if (containerNode.infosetWalkerBlockCount > 0) {
+ // This happens in two cases:
+ //
+ // 1) If we have already walked into a complex type that includes an
+ // unordered sequence. When we start adding the unordered sequence we
+ // increment the infosetWalkerBlockCount of the container because we
+ // can't increment the infosetWalkerBlockCount of the yet to be
+ // created unordered sequence elements. So we take no further steps
+ // until the unordered sequence is finished and the block is removed
+ // 2) When we are speculatively parsing elements. When an element may or
+ // may not exist, we set a point of uncertainty before it is created.
+ // Setting this mark increments the block count of the container
+ // element because this optional element does not exist yet.
+ //
+ // In both cases we cannot take a step until the block is removed
+ false
+
+ } else {
+ // no blocks on the container, figure out if we can take a step for the
+ // element and the child index of this container
+
+ val children = containerNode.contents
+ val childIndex = containerIndexStack.top
+
+ if (childIndex < children.size) {
+ // There is a child element at this index. Taking a step would create
+ // the events for, and moved passed, this element.
+ val elem = children(childIndex)
+ if (elem.infosetWalkerBlockCount > 0) {
+ // This element has a block associated with it, likely meaning we are
+ // speculatively parsing this element and so it may or may not exist.
+ // We cannot walk into it until we figure it out and the block is
+ // removed.
+ false
+ } else if (elem.isInstanceOf[DISimple] && !elem.isFinal) {
+ // This is a simple element that is not final, i.e. we are still
+ // doing some parsing for this simple element. Wait until we finish
+ // that parse before stepping into it.
+ false
+ } else {
+ true
+ }
+ } else {
+ // There is not currently a child at this index.
+ if (containerNode.isFinal) {
+ // This container is final, no more children will be added. So we
+ // can make a step that will end this container
+ true
+ } else {
+ // This container is not final, meaning we are potentially waiting
+ // for more another child to be added at this index. In this case, we
+ // cannot make a step until a child is added or the container is
+ // marked as final
+ false
+ }
+ }
+ }
+ }
+
+ private trait InfosetWalkerStep {
+ /**
+ * Output events associated with this step kind, and mutate the
+ * InfosetWalker state to walk to the next node in the infoset
+ */
+ def step(): Unit
+
+ final def moveToFirstChild(newContainer: DINode): Unit = {
+ containerNode = newContainer
+ containerIndexStack.push(0)
+ }
+
+ final def moveToContainer(): Unit = {
+ containerNode = containerNode.containerNode
+ containerIndexStack.pop
+ }
+
+ final def moveToNextSibling(): Unit = {
+ containerIndexStack.push(containerIndexStack.pop + 1)
+ }
+ }
+
+ private object InfosetWalkerStepStart extends InfosetWalkerStep {
+ /**
+ * Start the document. Note that because the top of container index is
+ * initialized to one less that the starting index, we also call
+ * moveToNextSibling to increment the starting index to the correct
+ * position.
+ */
+ override def step(): Unit = {
+ outputter.startDocument()
+ moveToNextSibling()
+ }
+ }
+
+ private object InfosetWalkerStepEnd extends InfosetWalkerStep {
+ /**
+ * End document and clean up state. Setting finished to true causes
+ * the next step to be None, walk() will return, and the caller
+ * should not call walk() again because it is finished.
+ */
+ override def step(): Unit = {
+ outputter.endDocument()
+ containerNode = null
+ containerIndexStack = null
+ finished = true
+ }
+ }
+
+ private object InfosetWalkerStepMove extends InfosetWalkerStep {
+ /**
+ * Output start/end events for DIComplex/DIArray/DISimple, and mutate state
+ * so we are looking at the next node in the infoset.
+ */
+ def step(): Unit = {
+ val children = containerNode.contents
+ val childIndex = containerIndexStack.top
+
+ if (childIndex < children.size) {
+ // This block means we need to create a start event for the element in
+ // the children array at childIndex. We then need to mutate the walker
+ // state so the next call to step() is for either the first child of
+ // this element or the next sibling.
+
+ children(childIndex) match {
+ case s: DISimple => {
+ if (!s.isHidden || walkHidden) {
+ outputter.startSimple(s)
+ outputter.endSimple(s)
+ }
+ if (removeUnneeded) {
+ // now we can remove this simple element to free up memory
+ containerNode.freeChildIfNoLongerNeeded(containerIndexStack.top)
+ }
+ moveToNextSibling()
+ }
+ case c: DIComplex => {
+ if (!c.isHidden || walkHidden) {
+ outputter.startComplex(c)
+ moveToFirstChild(c)
+ } else {
+ moveToNextSibling()
+ }
+ }
+ case a: DIArray => {
+ if (!a.isHidden || walkHidden) {
+ outputter.startArray(a)
+ moveToFirstChild(a)
+ } else {
+ moveToNextSibling()
+ }
+ }
+ }
+
+ } else {
+ // This else block means that we incremented the index stack past the
+ // number of children in this container (must be a DIComplex/DIDocument
+ // or DIArray), which means we have created events for all if its
+ // children. So we must now create the appropriate end event for the
+ // container and then mutate the state so that we are looking at its
+ // next sibling. Note that if there is no next sibling, that will be
+ // found the next time step() is called (because we incremented past
+ // this element) and we will fall into this block again, call the end
+ // function again, and repeat the process.
+
+ // create appropriate end event
+ containerNode match {
+ case a: DIArray => {
+ outputter.endArray(a)
+ }
+ case c: DIComplex => {
+ outputter.endComplex(c)
+ }
+ case _ => Assert.impossible()
+ }
+
+ // we've ended this array/complex associated with the container, so we
+ // now want to move to the parent container, potentially free up the
+ // memory associated with this container, and then move to the next
+ // sibling of this container
+ moveToContainer()
+ if (removeUnneeded) {
+ containerNode.freeChildIfNoLongerNeeded(containerIndexStack.top)
+ }
+ moveToNextSibling()
+ }
+ }
+ }
+
+}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/TeeInfosetOutputter.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/TeeInfosetOutputter.scala
new file mode 100644
index 0000000..c2ed4f0
--- /dev/null
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/TeeInfosetOutputter.scala
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.infoset
+
+
+/**
+ * Receive infoset events and forward them to one or more InfosetOutputters.
+ * For infoset events that return a boolean, this returns true only if all
+ * outputters return true, otherwise false is returned. Additionally, all
+ * events are called on all the outputters regardless of the return of any
+ * previous outputters.
+ *
+ * @param outputters
+ *
+ * InfosetOutputters to send all events
+ */
+class TeeInfosetOutputter(outputters: InfosetOutputter*)
+ extends InfosetOutputter {
+
+ override def reset(): Unit = {
+ outputters.foreach { _.reset() }
+ }
+
+ override def startSimple(simple: DISimple): Boolean = {
+ outputters.foldLeft(true) { case (res, outputter) =>
+ res & outputter.startSimple(simple)
+ }
+ }
+
+ override def endSimple(simple: DISimple): Boolean = {
+ outputters.foldLeft(true) { case (res, outputter) =>
+ res & outputter.endSimple(simple)
+ }
+ }
+
+ override def startComplex(complex: DIComplex): Boolean = {
+ outputters.foldLeft(true) { case (res, outputter) =>
+ res & outputter.startComplex(complex)
+ }
+ }
+
+ override def endComplex(complex: DIComplex): Boolean = {
+ outputters.foldLeft(true) { case (res, outputter) =>
+ res & outputter.endComplex(complex)
+ }
+ }
+
+ override def startArray(array: DIArray): Boolean = {
+ outputters.foldLeft(true) { case (res, outputter) =>
+ res & outputter.startArray(array)
+ }
+ }
+
+ override def endArray(array: DIArray): Boolean = {
+ outputters.foldLeft(true) { case (res, outputter) =>
+ res & outputter.endArray(array)
+ }
+ }
+
+ override def startDocument(): Boolean = {
+ outputters.foldLeft(true) { case (res, outputter) =>
+ res & outputter.startDocument()
+ }
+ }
+
+ override def endDocument(): Boolean = {
+ outputters.foldLeft(true) { case (res, outputter) =>
+ res & outputter.endDocument()
+ }
+ }
+}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/XMLTextInfosetOutputter.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/XMLTextInfosetOutputter.scala
index 6f36104..6dbd1a0 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/XMLTextInfosetOutputter.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/XMLTextInfosetOutputter.scala
@@ -47,15 +47,19 @@ class XMLTextInfosetOutputter private (writer: java.io.Writer, pretty: Boolean,
resetIndentation()
}
- private def getTagName(elem: DIElement): String = {
+ private def outputTagName(elem: DIElement): Unit = {
val prefix = elem.erd.thisElementsNamespacePrefix
- if (prefix == null || prefix == "") elem.erd.name else prefix + ":" + elem.erd.name
+ if (prefix != null && prefix != "") {
+ writer.write(prefix)
+ writer.write(":")
+ }
+ writer.write(elem.erd.name)
}
- private def outputStartTag(elem: DIElement, name: String, isEmpty: Boolean=false): Unit = {
+ private def outputStartTag(elem: DIElement): Unit = {
writer.write("<")
- writer.write(name)
+ outputTagName(elem)
val nsbStart = elem.erd.minimizedScope
val nsbEnd = if (elem.isRoot) scala.xml.TopScope else elem.diParent.erd.minimizedScope
@@ -69,23 +73,21 @@ class XMLTextInfosetOutputter private (writer: java.io.Writer, pretty: Boolean,
writer.write(" xsi:nil=\"true\"")
}
- if (isEmpty) {
- writer.write(" />")
- } else {
- writer.write(">")
- }
+ writer.write(">")
}
- private def outputEndTag(elem: DIElement, name: String): Unit = {
+ private def outputEndTag(elem: DIElement): Unit = {
writer.write("</")
- writer.write(name)
+ outputTagName(elem)
writer.write(">")
}
override def startSimple(simple: DISimple): Boolean = {
- if (pretty) outputIndentation(writer)
- val name = getTagName(simple)
- outputStartTag(simple, name)
+ if (pretty) {
+ writer.write(System.lineSeparator())
+ outputIndentation(writer)
+ }
+ outputStartTag(simple)
if (!isNilled(simple) && simple.hasValue) {
val text =
@@ -99,8 +101,7 @@ class XMLTextInfosetOutputter private (writer: java.io.Writer, pretty: Boolean,
writer.write(text)
}
- outputEndTag(simple, name)
- if (pretty) writer.write(System.lineSeparator())
+ outputEndTag(simple)
true
}
@@ -110,26 +111,24 @@ class XMLTextInfosetOutputter private (writer: java.io.Writer, pretty: Boolean,
}
override def startComplex(complex: DIComplex): Boolean = {
- val name = getTagName(complex)
- if (pretty) outputIndentation(writer)
- outputStartTag(complex, name, !complex.hasVisibleChildren)
- if (pretty) writer.write(System.lineSeparator())
+ if (pretty) {
+ writer.write(System.lineSeparator())
+ outputIndentation(writer)
+ }
+ outputStartTag(complex)
incrementIndentation()
true
}
override def endComplex(complex: DIComplex): Boolean = {
decrementIndentation()
- if (!complex.hasVisibleChildren) {
- // Empty complex elements should have an inline closing tag
- true
- } else {
- val name = getTagName(complex)
- if (pretty) outputIndentation(writer)
- outputEndTag(complex, name)
- if (pretty) writer.write(System.lineSeparator())
- true
+ if (pretty && complex.hasVisibleChildren) {
+ // only output newline and indentation for non-empty complex types
+ writer.write(System.lineSeparator())
+ outputIndentation(writer)
}
+ outputEndTag(complex)
+ true
}
override def startArray(array: DIArray): Boolean = {
@@ -143,12 +142,12 @@ class XMLTextInfosetOutputter private (writer: java.io.Writer, pretty: Boolean,
}
override def startDocument(): Boolean = {
- writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>")
- if (pretty) writer.write(System.lineSeparator())
+ writer.write("""<?xml version="1.0" encoding="UTF-8"?>""")
true
}
override def endDocument(): Boolean = {
+ writer.write(System.lineSeparator())
writer.flush()
true
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DataProcessor.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DataProcessor.scala
index b6b190d..a469a4f 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DataProcessor.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DataProcessor.scala
@@ -18,53 +18,57 @@
package org.apache.daffodil.processors
import java.io.File
+import java.io.IOException
import java.io.ObjectOutputStream
-import java.nio.channels.Channels
import java.nio.CharBuffer
-import java.nio.file.Files
import java.nio.LongBuffer
+import java.nio.channels.Channels
+import java.nio.file.Files
import java.util.zip.GZIPOutputStream
+import scala.collection.immutable.Queue
+
import org.xml.sax.ErrorHandler
import org.xml.sax.SAXException
import org.xml.sax.SAXParseException
-import org.apache.daffodil.Implicits._
-import scala.collection.immutable.Queue; object INoWarn4 { ImplicitsSuppressUnusedImportWarning() }
-import org.apache.daffodil.equality._; object EqualityNoWarn3 { EqualitySuppressUnusedImportWarning() }
-import org.apache.daffodil.api.WithDiagnostics
-import org.apache.daffodil.exceptions.Assert
-import org.apache.daffodil.dsom._
import org.apache.daffodil.ExecutionMode
import org.apache.daffodil.api.DFDL
-import org.apache.daffodil.api.WithDiagnostics
-import org.apache.daffodil.util.Validator
+import org.apache.daffodil.api.DaffodilTunables
import org.apache.daffodil.api.ValidationMode
-import org.apache.daffodil.externalvars.ExternalVariablesLoader
-import org.apache.daffodil.externalvars.Binding
-import org.apache.daffodil.util.Maybe
-import org.apache.daffodil.util.Maybe._
-import org.apache.daffodil.util.Logging
+import org.apache.daffodil.api.WithDiagnostics
import org.apache.daffodil.debugger.Debugger
-import org.apache.daffodil.processors.unparsers.UState
-import org.apache.daffodil.infoset.InfosetInputter
-import org.apache.daffodil.processors.unparsers.UnparseError
-import org.apache.daffodil.oolag.ErrorAlreadyHandled
+import org.apache.daffodil.dsom.TunableLimitExceededError
+import org.apache.daffodil.dsom._
+import org.apache.daffodil.equality._; object EqualityNoWarn3 { EqualitySuppressUnusedImportWarning() }
import org.apache.daffodil.events.MultipleEventHandler
-import org.apache.daffodil.io.DirectOrBufferedDataOutputStream
-import org.apache.daffodil.io.InputSourceDataInputStream
-import org.apache.daffodil.util.LogLevel
+import org.apache.daffodil.exceptions.Assert
+import org.apache.daffodil.exceptions.UnsuppressableException
+import org.apache.daffodil.externalvars.Binding
+import org.apache.daffodil.externalvars.ExternalVariablesLoader
+import org.apache.daffodil.infoset.DIElement
+import org.apache.daffodil.infoset.InfosetElement
+import org.apache.daffodil.infoset.InfosetException
+import org.apache.daffodil.infoset.InfosetInputter
+import org.apache.daffodil.infoset.InfosetOutputter
+import org.apache.daffodil.infoset.TeeInfosetOutputter
+import org.apache.daffodil.infoset.XMLTextInfosetOutputter
import org.apache.daffodil.io.BitOrderChangeException
+import org.apache.daffodil.io.DirectOrBufferedDataOutputStream
import org.apache.daffodil.io.FileIOException
-import org.apache.daffodil.infoset._
+import org.apache.daffodil.io.InputSourceDataInputStream
+import org.apache.daffodil.oolag.ErrorAlreadyHandled
+import org.apache.daffodil.processors.parsers.PState
import org.apache.daffodil.processors.parsers.ParseError
import org.apache.daffodil.processors.parsers.Parser
-import org.apache.daffodil.processors.parsers.PState
-import org.apache.daffodil.exceptions.UnsuppressableException
-import org.apache.daffodil.dsom.TunableLimitExceededError
-import org.apache.daffodil.api.DaffodilTunables
-import java.io.IOException
+import org.apache.daffodil.processors.unparsers.UState
+import org.apache.daffodil.processors.unparsers.UnparseError
+import org.apache.daffodil.util.LogLevel
+import org.apache.daffodil.util.Logging
+import org.apache.daffodil.util.Maybe
+import org.apache.daffodil.util.Maybe._
import org.apache.daffodil.util.Misc
+import org.apache.daffodil.util.Validator
/**
* Implementation mixin - provides simple helper methods
@@ -389,12 +393,28 @@ class DataProcessor private (
def parse(input: InputSourceDataInputStream, output: InfosetOutputter): DFDL.ParseResult = {
Assert.usage(!this.isError)
+ // If full validation is enabled, tee all the infoset events to a second
+ // infoset outputter that writes the infoset to a byte array, and then
+ // we'll validate that byte array upon a successful parse.
+ //
+ // TODO: ideally we could create a validator that validates using only SAX
+ // events from a ContentHandler. Then we could just validate as parse
+ // events are created rather than writing the entire infoset in memory and
+ // then validating at the end of the parse. See DAFFODIL-2386
+ //
+ val (outputter, maybeValidationBytes) =
+ if (validationMode == ValidationMode.Full) {
+ val bos = new java.io.ByteArrayOutputStream()
+ val xmlOutputter = new XMLTextInfosetOutputter(bos, false)
+ val teeOutputter = new TeeInfosetOutputter(output, xmlOutputter)
+ (teeOutputter, One(bos))
+ } else {
+ (output, Nope)
+ }
+
val rootERD = ssrd.elementRuntimeData
- val initialState = PState.createInitialPState(rootERD, input, output, this)
- parse(initialState)
- }
+ val state = PState.createInitialPState(rootERD, input, outputter, this, areDebugging)
- def parse(state: PState): ParseResult = {
ExecutionMode.usingRuntimeMode {
if (areDebugging) {
@@ -406,15 +426,21 @@ class DataProcessor private (
doParse(ssrd.parser, state)
val pr = new ParseResult(this, state)
if (!pr.isProcessingError) {
- pr.validateResult()
- // now that everything has succeeded, call the infoset outputter to
- // output the infoset
- //
- // TODO: eventually, we want infoset outputting to happen while parsing
- // so that we can start to throw away infoset nodes. When that happens
- // we can remove this line.
- state.infoset.visit(state.output)
+ // By the time we get here, all infoset nodes have been setFinal, all
+ // walker blocks released, and all elements walked. The one exception
+ // is that the root node has not been set final because setFinal is
+ // handled by the sequence parser and there is no sequence around the
+ // root node. So mark it as final and do one last walk to end the
+ // document.
+ state.infoset.contents(0).setFinal()
+ state.walker.walk()
+ Assert.invariant(state.walker.isFinished)
+
+ if (maybeValidationBytes.isDefined) {
+ pr.validateResult(maybeValidationBytes.get.toByteArray)
+ }
+
state.output.setBlobPaths(state.blobPaths)
} else {
// failed, so delete all blobs that were created
@@ -660,26 +686,21 @@ class ParseResult(dp: DataProcessor, override val resultState: PState)
*
* @param state the initial parse state.
*/
- def validateResult(): Unit = {
+ def validateResult(bytes: Array[Byte]): Unit = {
Assert.usage(resultState.processorStatus eq Success)
- if (dp.validationMode == ValidationMode.Full) {
- val schemaURIStrings = resultState.infoset.asInstanceOf[InfosetElement].runtimeData.schemaURIStringsForFullValidation
- try {
- val bos = new java.io.ByteArrayOutputStream()
- val xml = new XMLTextInfosetOutputter(bos, false)
- resultState.infoset.visit(xml)
- val bis = new java.io.ByteArrayInputStream(bos.toByteArray)
- Validator.validateXMLSources(schemaURIStrings, bis, this)
- } catch {
- //
- // Some SAX Parse errors are thrown even if you specify an error handler to the
- // validator.
- //
- // So we also need this catch
- //
- case e: SAXException =>
- resultState.validationErrorNoContext(e)
- }
+ val schemaURIStrings = resultState.infoset.asInstanceOf[InfosetElement].runtimeData.schemaURIStringsForFullValidation
+ try {
+ val bis = new java.io.ByteArrayInputStream(bytes)
+ Validator.validateXMLSources(schemaURIStrings, bis, this)
+ } catch {
+ //
+ // Some SAX Parse errors are thrown even if you specify an error handler to the
+ // validator.
+ //
+ // So we also need this catch
+ //
+ case e: SAXException =>
+ resultState.validationErrorNoContext(e)
}
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/ElementKindParsers.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/ElementKindParsers.scala
index 3bec236..1e34290 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/ElementKindParsers.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/ElementKindParsers.scala
@@ -219,6 +219,8 @@ abstract class ChoiceDispatchCombinatorParserBase(rd: TermRuntimeData,
pstate.resetToPointOfUncertainty(pou)
}
+ val savedLastChildNode = pstate.infoset.contents.lastOption
+
// Note that we are intentionally not pushing/popping a new
// discriminator here, as is done in the ChoiceCombinatorParser and
// AltCompParser. This has the effect that if a branch of this direct
@@ -232,6 +234,24 @@ abstract class ChoiceDispatchCombinatorParserBase(rd: TermRuntimeData,
if (pstate.processorStatus eq Success) {
log(LogLevel.Debug, "Choice dispatch success: %s", parser)
+
+ // We usually rely on the sequence parser to set elements as final.
+ // But choices with scalar elements do not necessarily have a
+ // sequence surrounding them and so they aren't set final. In order
+ // to set these elements final, we do it here as well. We will
+ // attempt to walk the infoset after the PoU is discarded.
+ //
+ // Note that we must do a null check because it's possible there was
+ // a sequence, which figured out the element was final, walked it and
+ // it was removed.
+ val newLastChildNode = pstate.infoset.contents.lastOption
+ if (newLastChildNode != savedLastChildNode) {
+ val last = newLastChildNode.get
+ if (last != null) {
+ last.setFinal()
+ }
+ }
+
} else {
log(LogLevel.Debug, "Choice dispatch failed: %s", parser)
val diag = new ChoiceDispatchFailed(context.schemaFileLocation, pstate, pstate.diagnostics)
@@ -240,6 +260,7 @@ abstract class ChoiceDispatchCombinatorParserBase(rd: TermRuntimeData,
}
}
}
+ pstate.walker.walk()
}
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/ExpressionEvaluatingParsers.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/ExpressionEvaluatingParsers.scala
index e7c24a1..dde04ce 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/ExpressionEvaluatingParsers.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/ExpressionEvaluatingParsers.scala
@@ -120,6 +120,7 @@ trait WithDetachedParser {
res
}
+ pstate.walker.walk()
pstate.setParent(priorElement)
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/NilEmptyCombinatorParsers.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/NilEmptyCombinatorParsers.scala
index c7f9c64..9cf8a93 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/NilEmptyCombinatorParsers.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/NilEmptyCombinatorParsers.scala
@@ -43,6 +43,7 @@ abstract class NilOrValueParser(ctxt: TermRuntimeData, nilParser: Parser, valueP
// no-op. We found nil, withPointOfUncertainty will discard the pou
}
}
+ pstate.walker.walk()
}
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/PState.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/PState.scala
index 11cbd0c..6d90d76 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/PState.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/PState.scala
@@ -35,6 +35,7 @@ import org.apache.daffodil.infoset.DISimple
import org.apache.daffodil.infoset.Infoset
import org.apache.daffodil.infoset.InfosetDocument
import org.apache.daffodil.infoset.InfosetOutputter
+import org.apache.daffodil.infoset.InfosetWalker
import org.apache.daffodil.io.InputSourceDataInputStream
import org.apache.daffodil.io.DataInputStream
import org.apache.daffodil.processors.DataLoc
@@ -151,7 +152,7 @@ class MPState private () {
final class PState private (
var infoset: DIElement,
var dataInputStream: InputSourceDataInputStream,
- val output: InfosetOutputter,
+ val walker: InfosetWalker,
vmap: VariableMap,
diagnosticsArg: List[Diagnostic],
val mpstate: MPState,
@@ -163,6 +164,8 @@ final class PState private (
override def currentNode = Maybe(infoset)
+ def output = walker.outputter
+
/**
* This stack is used to track points of uncertainty during a parse. When a
* parser determines a PoU should exist, it should call withPointOfUncertainty
@@ -245,6 +248,7 @@ final class PState private (
changedVariablesStack.push(mutable.MutableList[GlobalQName]())
val m = markPool.getFromPool(requestorID)
m.captureFrom(this, requestorID, context)
+ m.element.infosetWalkerBlockCount += 1
m
}
@@ -256,6 +260,7 @@ final class PState private (
private def reset(m: PState.Mark): Unit = {
// threadCheck()
+ m.element.infosetWalkerBlockCount -= 1
m.restoreInto(this)
m.clear()
markPool.returnToPool(m)
@@ -268,6 +273,7 @@ final class PState private (
}
private def discard(m: PState.Mark): Unit = {
+ m.element.infosetWalkerBlockCount -= 1
dataInputStream.discard(m.disMark)
m.clear()
markPool.returnToPool(m)
@@ -541,6 +547,7 @@ object PState {
val simpleElementState = DISimpleState()
val complexElementState = DIComplexState()
+ var element: DIElement = _
var disMark: DataInputStream.Mark = _
var variableMap: VariableMap = _
var processorStatus: ProcessorResult = _
@@ -567,11 +574,11 @@ object PState {
}
def captureFrom(ps: PState, requestorID: String, context: RuntimeData): Unit = {
- val e = ps.thisElement
- if (e.isSimple)
- simpleElementState.captureFrom(e)
+ this.element = ps.thisElement
+ if (element.isSimple)
+ simpleElementState.captureFrom(element)
else
- complexElementState.captureFrom(e)
+ complexElementState.captureFrom(element)
this.disMark = ps.dataInputStream.mark(requestorID)
this.variableMap = ps.variableMap
this.processorStatus = ps.processorStatus
@@ -583,10 +590,10 @@ object PState {
}
def restoreInfoset(ps: PState) = {
- val e = ps.thisElement
- e match {
- case s: DISimple => simpleElementState.restoreInto(e)
- case c: DIComplex => complexElementState.restoreInto(e)
+ Assert.invariant(this.element eq ps.thisElement)
+ this.element match {
+ case s: DISimple => simpleElementState.restoreInto(this.element)
+ case c: DIComplex => complexElementState.restoreInto(this.element)
}
}
@@ -628,11 +635,18 @@ object PState {
root: ElementRuntimeData,
dis: InputSourceDataInputStream,
output: InfosetOutputter,
- dataProc: DFDL.DataProcessor): PState = {
+ dataProc: DFDL.DataProcessor,
+ areDebugging: Boolean): PState = {
val tunables = dataProc.getTunables()
val doc = Infoset.newDocument(root).asInstanceOf[DIElement]
- createInitialPState(doc.asInstanceOf[InfosetDocument], root, dis, output, dataProc)
+ createInitialPState(
+ doc.asInstanceOf[InfosetDocument],
+ root,
+ dis,
+ output,
+ dataProc,
+ areDebugging)
}
/**
@@ -643,7 +657,8 @@ object PState {
root: ElementRuntimeData,
dis: InputSourceDataInputStream,
output: InfosetOutputter,
- dataProc: DFDL.DataProcessor): PState = {
+ dataProc: DFDL.DataProcessor,
+ areDebugging: Boolean): PState = {
/**
* This is a full deep copy as variableMap is mutable. Reusing
@@ -654,10 +669,25 @@ object PState {
val diagnostics = Nil
val mutablePState = MPState()
val tunables = dataProc.getTunables()
+ val infosetWalker = InfosetWalker(
+ doc.asInstanceOf[DIElement],
+ output,
+ walkHidden = false,
+ ignoreBlocks = false,
+ removeUnneeded = !areDebugging)
dis.cst.setPriorBitOrder(root.defaultBitOrder)
- val newState = new PState(doc.asInstanceOf[DIElement], dis, output, variables, diagnostics, mutablePState,
- dataProc.asInstanceOf[DataProcessor], Nope, Seq.empty, tunables)
+ val newState = new PState(
+ doc.asInstanceOf[DIElement],
+ dis,
+ infosetWalker,
+ variables,
+ diagnostics,
+ mutablePState,
+ dataProc.asInstanceOf[DataProcessor],
+ Nope,
+ Seq.empty,
+ tunables)
newState
}
@@ -668,9 +698,10 @@ object PState {
root: ElementRuntimeData,
data: String,
output: InfosetOutputter,
- dataProc: DFDL.DataProcessor): PState = {
+ dataProc: DFDL.DataProcessor,
+ areDebugging: Boolean): PState = {
val in = InputSourceDataInputStream(data.getBytes("utf-8"))
- createInitialPState(root, in, output, dataProc)
+ createInitialPState(root, in, output, dataProc, areDebugging)
}
/**
@@ -680,8 +711,9 @@ object PState {
root: ElementRuntimeData,
input: java.nio.channels.ReadableByteChannel,
output: InfosetOutputter,
- dataProc: DFDL.DataProcessor): PState = {
+ dataProc: DFDL.DataProcessor,
+ areDebugging: Boolean): PState = {
val dis = InputSourceDataInputStream(Channels.newInputStream(input))
- createInitialPState(root, dis, output, dataProc)
+ createInitialPState(root, dis, output, dataProc, areDebugging)
}
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/Parser.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/Parser.scala
index 540b4a4..087efcd 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/Parser.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/Parser.scala
@@ -190,6 +190,8 @@ class ChoiceParser(
var successfullyParsedChildBranch = false
+ val savedLastChildNode = pstate.infoset.contents.lastOption
+
while (!successfullyParsedChildBranch && i < numAlternatives) {
val parser = childParsers(i)
@@ -203,6 +205,24 @@ class ChoiceParser(
// Choice branch was successfull. Break out of the loop and let
// withPointOfUncertainty discard the pou
successfullyParsedChildBranch = true
+
+ // We usually rely on the sequence parser to set elements as final.
+ // But choices with scalar elements do not necessarily have a
+ // sequence surrounding them and so they aren't set final. In order
+ // to set these elements final, we do it here as well. We will
+ // attempt to walk the infoset after the PoU is discarded.
+ //
+ // Note that we must do a null check because it's possible there was
+ // a sequence, which figured out the element was final, walked it and
+ // it was removed.
+ val newLastChildNode = pstate.infoset.contents.lastOption
+ if (newLastChildNode != savedLastChildNode) {
+ val last = newLastChildNode.get
+ if (last != null) {
+ last.setFinal()
+ }
+ }
+
} else {
// Failed to parse this branch alternative. Create diagnostic and
// check if anything resolved the associated point of uncertainty
@@ -225,6 +245,8 @@ class ChoiceParser(
}
}
}
+
+ pstate.walker.walk()
}
if (!successfullyParsedChildBranch) {
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SequenceParserBases.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SequenceParserBases.scala
index 7389ab6..6415cfb 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SequenceParserBases.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SequenceParserBases.scala
@@ -19,6 +19,7 @@ package org.apache.daffodil.processors.parsers
import org.apache.daffodil.dsom.TunableLimitExceededError
import org.apache.daffodil.exceptions.Assert
+import org.apache.daffodil.infoset.DIArray
import org.apache.daffodil.infoset.DIComplex
import org.apache.daffodil.processors.ElementRuntimeData
import org.apache.daffodil.processors.Evaluatable
@@ -82,6 +83,18 @@ abstract class SequenceParserBase(
val infosetIndexStart = pstate.infoset.asInstanceOf[DIComplex].childNodes.size
+ if (!isOrdered) {
+ // If this is an unordered sequence, upon completion of parsing all the
+ // elements we will reorder the elements into schema definition order.
+ // This means that we cannot let the infoset walker walk any of the
+ // elements while we are parsing because their event order might change.
+ // To ensure we don't walk, add a block on the parent of the infoset
+ // elements. The infoset walker will inspect this to see if it should
+ // walk any children. We'll remove this block once the unordered sequence
+ // is complete.
+ pstate.infoset.infosetWalkerBlockCount += 1
+ }
+
/**
* On exit from the sequence loop, if the last thing was Missing, we
* want to look back one prior to see if that followed a EmptyRep or AbsentRep,
@@ -97,7 +110,13 @@ abstract class SequenceParserBase(
// This loop iterates over the children terms of the sequence
//
while (!isDone && (scpIndex < limit) && (pstate.processorStatus eq Success)) {
+
+ // keep track of the current last child node. If the last child changes
+ // while parsing, we know a new child was added in this loop
+ val savedLastChildNode = pstate.infoset.contents.lastOption
+
child = children(scpIndex).asInstanceOf[SequenceChildParser]
+
child match {
case parser: RepeatingChildParser => {
//
@@ -183,6 +202,35 @@ abstract class SequenceParserBase(
//
pstate.mpstate.moveOverOneGroupIndexOnly()
}
+
+ val newLastChildNode = pstate.infoset.contents.lastOption
+ if (newLastChildNode != savedLastChildNode) {
+ // We have added at least one child to to this complex during
+ // this array loop.
+ //
+ // If the new child is a DIArray, we know this DIArray has at
+ // least one element, but we don't know if we actually added a
+ // new one in this loop or not. So just get the last array
+ // element and set it as final anyways. Note that we need a null
+ // check because if we didn't add an array element this go
+ // around, the last element may have already been walked and
+ // freed and so could now be null.
+ //
+ // If it's not a DIArray, that means it's just an optional
+ // simple/complex and that will get set final below where all
+ // other non-array elements get set as final.
+ newLastChildNode.get match {
+ case a: DIArray => {
+ val lastArrayElem = a.contents.last
+ if (lastArrayElem != null) {
+ lastArrayElem.setFinal()
+ pstate.walker.walk()
+ }
+ }
+ case _ => // non-array, that gets handled down below
+ }
+ }
+
} // end while for each repeat
parser.endArray(pstate)
} // end match case RepeatingChildParser
@@ -237,14 +285,76 @@ abstract class SequenceParserBase(
pstate.mpstate.moveOverOneGroupIndexOnly()
} // end case scalarParser
} // end match case parser
- if (isOrdered)
+
+ // now that we have finished parsing a single instance of this sequence,
+ // we need to potentially set things as final, get the last child to
+ // determine if it changed from the saved last child, which lets us know
+ // if a new child was actually added.
+ val newLastChildNode = pstate.infoset.contents.lastOption
+
+ if (!isOrdered) {
+ // In the special case of unordered sequences with arrays, the
+ // childParser is not a RepeatingChildParser, so array elements aren't
+ // setFinal above like normal arrays are a. Instead we parse one
+ // instance at a time in this loop.
+ //
+ // So if this the new last child node is a DIArray, we must set new
+ // array elements as final here. We can't know if we actually added a
+ // new DIArray element or not, so just set the last one as final
+ // regardless.
+ //
+ // Note that we do not need to do a null check because in an unordered
+ // sequence we are blocking, so we can't possible walk/free any of
+ // these newly added elements.
+ //
+ // Also note that the DIArray itself is set as final right below this.
+ // Again, because unordred sequences get blocked, that array won't be
+ // walked even though it's final.
+ newLastChildNode match {
+ case Some(a: DIArray) => a.contents.last.setFinal()
+ case _ => // do nothing
+ }
+ }
+
+ // We finished parsing one part of a sequence, which could either be an
+ // array, simple or complex. If the last child is different from when we
+ // started that means we must have added a new element and we can set it
+ // final and walk. Note that we must do a null check because it's
+ // possible this last child was already determined to be final, walked,
+ // and freed in the ChoiceParser.
+ if (newLastChildNode != savedLastChildNode) {
+ val lastChild = newLastChildNode.get
+ if (lastChild != null) {
+ lastChild.setFinal()
+ pstate.walker.walk()
+ }
+ }
+
+ if (isOrdered) {
+ // only increment scpIndex for ordered sequences. For unordered
+ // sequences, we just parse the single child parser repeatedly until we
+ // get a failure
scpIndex += 1
- else if (isDone) {
- val infoset = pstate.infoset.asInstanceOf[DIComplex]
- infoset.flattenAndValidateChildNodes(pstate, infosetIndexStart)
}
+
} // end while for each sequence child parser
+ if (!isOrdered) {
+ // we are unordered, so we need to reorder the new children into schema
+ // definition order, flatten arrays, and validate
+ val infoset = pstate.infoset.asInstanceOf[DIComplex]
+ infoset.flattenAndValidateChildNodes(pstate, infosetIndexStart)
+
+ // now that we have flattened children, we can decrement the block count
+ // that we incremented above. This will allow the infoset walker to walk
+ // into the new children that are now in the correct order.
+ pstate.infoset.infosetWalkerBlockCount -= 1
+
+ // we've unblocked the unordered sequence, try walking to output
+ // everything we've created
+ pstate.walker.walk()
+ }
+
if (child ne null) child.finalChecks(pstate, resultOfTry, priorResultOfTry)
pstate.mpstate.groupIndexStack.pop()
()
@@ -266,9 +376,11 @@ abstract class SequenceParserBase(
!roStatus.isInstanceOf[RequiredOptionalStatus.Required]
if (needsPoU) {
- pstate.withPointOfUncertainty("SequenceParserBase", parser.context) { pou =>
+ val ans = pstate.withPointOfUncertainty("SequenceParserBase", parser.context) { pou =>
parseOneInstanceWithMaybePoU(parser, pstate, roStatus, One(pou))
}
+ pstate.walker.walk()
+ ans
} else {
parseOneInstanceWithMaybePoU(parser, pstate, roStatus, Nope)
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/UnseparatedSequenceParsers.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/UnseparatedSequenceParsers.scala
index eed57e4..55c89b7 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/UnseparatedSequenceParsers.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/UnseparatedSequenceParsers.scala
@@ -27,7 +27,8 @@ trait Unseparated { self: SequenceChildParser =>
final def parseOne(pstate: PState, requiredOptional: RequiredOptionalStatus): ParseAttemptStatus = {
val prevBitPosBeforeChild = pstate.bitPos0b
self.childParser.parse1(pstate)
- parseResultHelper.computeParseAttemptStatus(self, prevBitPosBeforeChild, pstate, requiredOptional)
+ val res = parseResultHelper.computeParseAttemptStatus(self, prevBitPosBeforeChild, pstate, requiredOptional)
+ res
}
final def isPositional = true