You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@daffodil.apache.org by GitBox <gi...@apache.org> on 2018/12/11 23:23:16 UTC
[GitHub] mbeckerle closed pull request #149: Daffodil 2034 pools
mbeckerle closed pull request #149: Daffodil 2034 pools
URL: https://github.com/apache/incubator-daffodil/pull/149
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/daffodil-lib/src/main/resources/org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd b/daffodil-lib/src/main/resources/org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd
index 093d8e393..4b6a4a013 100644
--- a/daffodil-lib/src/main/resources/org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd
+++ b/daffodil-lib/src/main/resources/org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd
@@ -23,7 +23,15 @@
<xs:annotation>
<xs:appinfo source="http://www.ogf.org/dfdl/">
- <dfdl:defineFormat name="GeneralFormat">
+ <!--
+ This set of properties should never change. It can be added to
+ if properties are missing, but should not be changed.
+
+ Changes should be introduced by way of defining new formats in
+ terms of this one, which override definitions. See below in
+ the file for examples.
+ -->
+ <dfdl:defineFormat name="GeneralFormatOriginal">
<dfdl:format
alignment="1"
alignmentUnits="bytes"
@@ -93,6 +101,17 @@
utf16Width="fixed"
/>
</dfdl:defineFormat>
+
+ <dfdl:defineFormat name="GeneralFormat">
+ <dfdl:format ref="GeneralFormatOriginal" />
+ </dfdl:defineFormat>
+
+ <dfdl:defineFormat name="GeneralFormatForCrossTesting">
+ <dfdl:format ref="GeneralFormatOriginal"
+ calendarTimeZone=""
+ encodingErrorPolicy="error"
+ />
+ </dfdl:defineFormat>
</xs:appinfo>
</xs:annotation>
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/ProcessorStateBases.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/ProcessorStateBases.scala
index 114b82149..3b825d45b 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/ProcessorStateBases.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/ProcessorStateBases.scala
@@ -59,6 +59,10 @@ import org.apache.daffodil.schema.annotation.props.gen.ByteOrder
import org.apache.daffodil.processors.charset.BitsCharsetDecoder
import org.apache.daffodil.processors.charset.BitsCharsetEncoder
import org.apache.daffodil.processors.unparsers.UState
+import org.apache.daffodil.processors.dfa.Registers
+import org.apache.daffodil.processors.dfa.RegistersPool
+import org.apache.daffodil.processors.dfa.RegistersPool
+import org.apache.daffodil.processors.dfa.RegistersPool
/**
* Trait mixed into the PState.Mark object class and the ParseOrUnparseState
@@ -75,8 +79,7 @@ trait StateForDebugger {
def discriminator: Boolean = false
}
-case class TupleForDebugger(
- val bytePos: Long,
+case class TupleForDebugger(val bytePos: Long,
val childPos: Long,
val groupPos: Long,
val currentLocation: DataLocation,
@@ -119,8 +122,7 @@ trait SetProcessorMixin {
* which should be isolated to the alternative parser, and repParsers, i.e.,
* places where points-of-uncertainty are handled.
*/
-abstract class ParseOrUnparseState protected (
- protected var variableBox: VariableBox,
+abstract class ParseOrUnparseState protected (protected var variableBox: VariableBox,
var diagnostics: List[Diagnostic],
var dataProc: Maybe[DataProcessor],
val tunable: DaffodilTunables) extends DFDL.State
@@ -493,6 +495,17 @@ abstract class ParseOrUnparseState protected (
}
}
+ object dfaRegistersPool {
+ private val pool = new RegistersPool()
+
+ def getFromPool(requestorID: String) =
+ pool.getFromPool(requestorID)
+
+ def returnToPool(r: Registers) = pool.returnToPool(r)
+
+ def finalCheck() = pool.finalCheck
+ }
+
}
/**
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/Registers.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/Registers.scala
index 3b7ba7019..3abd4c6bf 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/Registers.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/Registers.scala
@@ -24,18 +24,7 @@ import org.apache.daffodil.util.Pool
import org.apache.daffodil.util.Poolable
import org.apache.daffodil.io.FormatInfo
-private[dfa] object TLRegistersPool extends ThreadLocal[RegistersPool] {
- override def initialValue = new RegistersPool()
-
- def pool() = this.get
-
- def getFromPool(requestorID: String) =
- pool.getFromPool(requestorID)
-
- def returnToPool(r: Registers) = pool.returnToPool(r)
-}
-
-private[dfa] class RegistersPool() extends Pool[Registers] {
+class RegistersPool() extends Pool[Registers] {
override def allocate = new Registers()
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextDelimitedParser.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextDelimitedParser.scala
index 67b484b36..53f493cd8 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextDelimitedParser.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextDelimitedParser.scala
@@ -29,9 +29,9 @@ import org.apache.daffodil.equality._
import org.apache.daffodil.util.MaybeChar
import org.apache.daffodil.processors.DelimiterIterator
import org.apache.daffodil.io.FormatInfo
+import org.apache.daffodil.processors.parsers.PState
-abstract class TextDelimitedParserBase(
- override val justificationTrim: TextJustificationType.Type,
+abstract class TextDelimitedParserBase(override val justificationTrim: TextJustificationType.Type,
override val parsingPadChar: MaybeChar,
override val context: TermRuntimeData)
extends DFAParser with PaddingRuntimeMixin {
@@ -39,12 +39,12 @@ abstract class TextDelimitedParserBase(
private lazy val padCharInfo = if (parsingPadChar.isDefined) parsingPadChar.toString else "NONE"
lazy val info: String = "justification='" + justificationTrim + "', padChar='" + padCharInfo + "'"
- final def parse(finfo: FormatInfo, input: DataInputStream, field: DFAField, delimIter: DelimiterIterator, isDelimRequired: Boolean): Maybe[ParseResult] = {
+ final def parse(finfo: PState, input: DataInputStream, field: DFAField, delimIter: DelimiterIterator, isDelimRequired: Boolean): Maybe[ParseResult] = {
Assert.invariant(field != null)
val lmt = new LongestMatchTracker()
- val fieldReg: Registers = TLRegistersPool.getFromPool("TextDelimitedParserBase1")
+ val fieldReg: Registers = finfo.dfaRegistersPool.getFromPool("TextDelimitedParserBase1")
fieldReg.reset(finfo, input, delimIter) // Initialization
@@ -66,13 +66,13 @@ abstract class TextDelimitedParserBase(
val d = delimIter.next()
input.resetPos(beforeDelimiter)
beforeDelimiter = input.markPos
- val delimReg: Registers = TLRegistersPool.getFromPool("TextDelimitedParserBase2")
+ val delimReg: Registers = finfo.dfaRegistersPool.getFromPool("TextDelimitedParserBase2")
delimReg.reset(finfo, input, delimIter)
d.run(delimReg)
if (delimReg.status == StateKind.Succeeded) {
lmt.successfulMatch(delimReg.matchStartPos, delimReg.delimString, d, delimIter.currentIndex)
}
- TLRegistersPool.returnToPool(delimReg)
+ finfo.dfaRegistersPool.returnToPool(delimReg)
}
if (!lmt.longestMatches.isEmpty) { stillSearching = false }
else {
@@ -137,8 +137,8 @@ abstract class TextDelimitedParserBase(
}
}
- TLRegistersPool.returnToPool(fieldReg)
- TLRegistersPool.pool.finalCheck
+ finfo.dfaRegistersPool.returnToPool(fieldReg)
+ finfo.dfaRegistersPool.finalCheck
result
}
@@ -149,8 +149,7 @@ abstract class TextDelimitedParserBase(
* Assumes that the delims DFAs were constructed with the Esc
* and EscEsc in mind.
*/
-class TextDelimitedParser(
- justArg: TextJustificationType.Type,
+class TextDelimitedParser(justArg: TextJustificationType.Type,
padCharArg: MaybeChar,
context: TermRuntimeData)
extends TextDelimitedParserBase(justArg, padCharArg, context) {
@@ -163,8 +162,7 @@ class TextDelimitedParser(
* Assumes that endBlock DFA was constructed with the
* EscEsc in mind.
*/
-class TextDelimitedParserWithEscapeBlock(
- justArg: TextJustificationType.Type,
+class TextDelimitedParserWithEscapeBlock(justArg: TextJustificationType.Type,
padCharArg: MaybeChar,
context: TermRuntimeData)
extends TextDelimitedParserBase(justArg, padCharArg, context) {
@@ -185,37 +183,37 @@ class TextDelimitedParserWithEscapeBlock(
}
}
- protected def removeLeftPadding(finfo: FormatInfo, input: DataInputStream, delimIter: DelimiterIterator): Unit = {
+ protected def removeLeftPadding(finfo: PState, input: DataInputStream, delimIter: DelimiterIterator): Unit = {
justificationTrim match {
case TextJustificationType.Center | TextJustificationType.Right if parsingPadChar.isDefined => {
- val leftPaddingRegister = TLRegistersPool.getFromPool("removeLeftPadding")
+ val leftPaddingRegister = finfo.dfaRegistersPool.getFromPool("removeLeftPadding")
leftPaddingRegister.reset(finfo, input, delimIter)
leftPadding.run(leftPaddingRegister)
- TLRegistersPool.returnToPool(leftPaddingRegister)
+ finfo.dfaRegistersPool.returnToPool(leftPaddingRegister)
}
case _ => // No left padding
}
}
- protected def removeRightPadding(finfo: FormatInfo, input: DataInputStream, delimIter: DelimiterIterator): Unit = {
+ protected def removeRightPadding(finfo: PState, input: DataInputStream, delimIter: DelimiterIterator): Unit = {
justificationTrim match {
case TextJustificationType.Center | TextJustificationType.Left if parsingPadChar.isDefined => {
- val rightPaddingRegister = TLRegistersPool.getFromPool("removeRightPadding")
+ val rightPaddingRegister = finfo.dfaRegistersPool.getFromPool("removeRightPadding")
rightPaddingRegister.reset(finfo, input, delimIter)
rightPadding.run(rightPaddingRegister)
- TLRegistersPool.returnToPool(rightPaddingRegister)
+ finfo.dfaRegistersPool.returnToPool(rightPaddingRegister)
}
case _ => // No right padding
}
}
- protected def parseStartBlock(finfo: FormatInfo, input: DataInputStream, startBlock: DFADelimiter, delimIter: DelimiterIterator): Boolean = {
- val startBlockRegister = TLRegistersPool.getFromPool("parseStartBlock")
+ protected def parseStartBlock(finfo: PState, input: DataInputStream, startBlock: DFADelimiter, delimIter: DelimiterIterator): Boolean = {
+ val startBlockRegister = finfo.dfaRegistersPool.getFromPool("parseStartBlock")
startBlockRegister.reset(finfo, input, delimIter)
startBlock.run(startBlockRegister) // find the block start, fail otherwise
val startStatus = startBlockRegister.status
- TLRegistersPool.returnToPool(startBlockRegister)
+ finfo.dfaRegistersPool.returnToPool(startBlockRegister)
startStatus match {
case StateKind.Succeeded => true // continue
case _ => false // Failed
@@ -226,14 +224,14 @@ class TextDelimitedParserWithEscapeBlock(
* Called to parse the rest of the field until we reach a block end, but
* beyond that, after we reach a block-end out until we reach the delimiter.
*/
- protected def parseRemainder(finfo: FormatInfo, input: DataInputStream,
+ protected def parseRemainder(finfo: PState, input: DataInputStream,
fieldEsc: DFAField,
startBlock: DFADelimiter, endBlock: DFADelimiter,
delimIter: DelimiterIterator, isDelimRequired: Boolean): Maybe[ParseResult] = {
val lmt = new LongestMatchTracker()
- val fieldRegister = TLRegistersPool.getFromPool("parseRemainder")
+ val fieldRegister = finfo.dfaRegistersPool.getFromPool("parseRemainder")
fieldRegister.reset(finfo, input, delimIter)
var stillSearching: Boolean = true
@@ -252,11 +250,11 @@ class TextDelimitedParserWithEscapeBlock(
case StateKind.Failed => stillSearching = false
case StateKind.Paused => {
// Pick up where field left off, we are looking for the blockEnd.
- val endBlockRegister = TLRegistersPool.getFromPool("parseRemainder2")
+ val endBlockRegister = finfo.dfaRegistersPool.getFromPool("parseRemainder2")
endBlockRegister.reset(finfo, input, delimIter)
endBlock.run(endBlockRegister)
val endBlockStatus = endBlockRegister.status
- TLRegistersPool.returnToPool(endBlockRegister)
+ finfo.dfaRegistersPool.returnToPool(endBlockRegister)
endBlockStatus match {
case StateKind.Succeeded => {
@@ -269,7 +267,7 @@ class TextDelimitedParserWithEscapeBlock(
while (delimIter.hasNext()) {
// Finally, we can look for the delimiter.
val d = delimIter.next() // Pick up where end of block/padding left off
- val delimRegister = TLRegistersPool.getFromPool("parseRemainder3")
+ val delimRegister = finfo.dfaRegistersPool.getFromPool("parseRemainder3")
input.resetPos(beforeDelimiter)
beforeDelimiter = input.markPos
delimRegister.reset(finfo, input, delimIter)
@@ -278,7 +276,7 @@ class TextDelimitedParserWithEscapeBlock(
if (delimRegister.status == StateKind.Succeeded) {
lmt.successfulMatch(delimRegister.matchStartPos, delimRegister.delimString, d, delimIter.currentIndex)
}
- TLRegistersPool.returnToPool(delimRegister)
+ finfo.dfaRegistersPool.returnToPool(delimRegister)
}
foundBlockEnd = true
stillSearching = false
@@ -344,11 +342,11 @@ class TextDelimitedParserWithEscapeBlock(
}
}
- TLRegistersPool.returnToPool(fieldRegister)
+ finfo.dfaRegistersPool.returnToPool(fieldRegister)
result
}
- def parse(finfo: FormatInfo, input: DataInputStream, field: DFAField, fieldEsc: DFAField,
+ def parse(finfo: PState, input: DataInputStream, field: DFAField, fieldEsc: DFAField,
startBlock: DFADelimiter, endBlock: DFADelimiter,
delimIter: DelimiterIterator, isDelimRequired: Boolean): Maybe[ParseResult] = {
Assert.invariant(fieldEsc != null)
@@ -363,7 +361,7 @@ class TextDelimitedParserWithEscapeBlock(
} else {
parseRemainder(finfo, input, fieldEsc, startBlock, endBlock, delimIter, isDelimRequired)
}
- TLRegistersPool.pool.finalCheck
+ finfo.dfaRegistersPool.finalCheck
res
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextDelimitedUnparser.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextDelimitedUnparser.scala
index 6f7ddf2a7..bdd478fc1 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextDelimitedUnparser.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextDelimitedUnparser.scala
@@ -86,7 +86,7 @@ class TextDelimitedUnparser(override val context: TermRuntimeData)
// We need to recognize the blockEnd in addition to the other pieces of
// text we should escape
//
- val fieldReg: Registers = TLRegistersPool.getFromPool("escapeBlock1")
+ val fieldReg: Registers = state.dfaRegistersPool.getFromPool("escapeBlock1")
val fieldEscapesIter = {
val ab = ArrayBuffer((blockEnd +: delims): _*)
@@ -128,7 +128,7 @@ class TextDelimitedUnparser(override val context: TermRuntimeData)
// We check for a blockEnd first, if it exists then we MUST
// generate an escape block
- val blockEndReg: Registers = TLRegistersPool.getFromPool("escapeBlock2")
+ val blockEndReg: Registers = state.dfaRegistersPool.getFromPool("escapeBlock2")
blockEndReg.reset(state, input, blockEndDelimIter)
blockEnd.run(blockEndReg)
val blockEndStatus = blockEndReg.status
@@ -136,7 +136,8 @@ class TextDelimitedUnparser(override val context: TermRuntimeData)
case StateKind.Succeeded if (!escapeEscapeChar.isDefined) => {
// Found an escapeEnd, which requires an escapeEscapeChar, but one was not provided
beforeDelimiter = DataInputStream.MarkPos.NoMarkPos
- UnparseError(One(context.schemaFileLocation),
+ UnparseError(
+ One(context.schemaFileLocation),
One(state.currentLocation),
"escapeEscapeCharacter was not defined but the escapeBlockEnd (%s) was present in the data.",
blockEnd.lookingFor)
@@ -171,7 +172,7 @@ class TextDelimitedUnparser(override val context: TermRuntimeData)
delimIter.reset()
while (delimIter.hasNext()) {
val d = delimIter.next()
- val delimReg: Registers = TLRegistersPool.getFromPool("escapeBlock3")
+ val delimReg: Registers = state.dfaRegistersPool.getFromPool("escapeBlock3")
input.resetPos(beforeDelimiter)
beforeDelimiter = input.markPos
delimReg.reset(state, input, delimIter)
@@ -187,7 +188,7 @@ class TextDelimitedUnparser(override val context: TermRuntimeData)
}
case _ => {
// this delim did not match, ignore it and discard its register
- TLRegistersPool.returnToPool(delimReg)
+ state.dfaRegistersPool.returnToPool(delimReg)
}
}
}
@@ -206,7 +207,7 @@ class TextDelimitedUnparser(override val context: TermRuntimeData)
val (_, matchedReg) = longestMatch(successes).get
val delim = matchedReg.delimString
fieldReg.appendToField(delim) // the delim just becomes field content, because we already had an escape block start.
- successes.foreach { case (d, r) => TLRegistersPool.returnToPool(r) }
+ successes.foreach { case (d, r) => state.dfaRegistersPool.returnToPool(r) }
successes.clear
shouldGenerateEscapeBlock = true
fieldReg.actionNum = 0
@@ -215,7 +216,7 @@ class TextDelimitedUnparser(override val context: TermRuntimeData)
// now go around the while loop again
} // end case StateKind.Failed for finding the block end.
} // end blockEndStatus.status match
- TLRegistersPool.returnToPool(blockEndReg)
+ state.dfaRegistersPool.returnToPool(blockEndReg)
} // end case StateKind.Paused for finding any of block end or a delimiter
} // end dfaStatus match
} // end while stillSearching
@@ -229,8 +230,8 @@ class TextDelimitedUnparser(override val context: TermRuntimeData)
// us to reuse the DFA for determining when to escape data while unparsing.
val resString = fieldReg.resultString.toString
- TLRegistersPool.returnToPool(fieldReg)
- TLRegistersPool.pool.finalCheck
+ state.dfaRegistersPool.returnToPool(fieldReg)
+ state.dfaRegistersPool.finalCheck
(resString, shouldGenerateEscapeBlock)
}
@@ -249,7 +250,7 @@ class TextDelimitedUnparser(override val context: TermRuntimeData)
Assert.invariant(field != null)
val successes: ArrayBuffer[(DFADelimiter, Registers)] = ArrayBuffer.empty
- val fieldReg: Registers = TLRegistersPool.getFromPool("escapeCharacter1")
+ val fieldReg: Registers = state.dfaRegistersPool.getFromPool("escapeCharacter1")
val delimIter = new AllDelimiterIterator(ArrayBuffer(delims: _*))
@@ -287,7 +288,7 @@ class TextDelimitedUnparser(override val context: TermRuntimeData)
delimIter.reset()
while (delimIter.hasNext()) {
val d = delimIter.next()
- val delimReg: Registers = TLRegistersPool.getFromPool("escapeCharacter2")
+ val delimReg: Registers = state.dfaRegistersPool.getFromPool("escapeCharacter2")
delimReg.reset(state, input, delimIter)
input.resetPos(beforeDelimiter)
beforeDelimiter = input.markPos
@@ -303,7 +304,7 @@ class TextDelimitedUnparser(override val context: TermRuntimeData)
}
case _ => {
// this delim did not match, ignore it and discard its register
- TLRegistersPool.returnToPool(delimReg)
+ state.dfaRegistersPool.returnToPool(delimReg)
}
}
}
@@ -335,7 +336,7 @@ class TextDelimitedUnparser(override val context: TermRuntimeData)
input.resetPos(beforeDelimiter)
Assert.invariant(input.skipChars(delim.length, state))
fieldReg.resetChars(state)
- successes.foreach { case (d, r) => TLRegistersPool.returnToPool(r) }
+ successes.foreach { case (d, r) => state.dfaRegistersPool.returnToPool(r) }
successes.clear
stillSearching = true
@@ -357,8 +358,8 @@ class TextDelimitedUnparser(override val context: TermRuntimeData)
// us to reuse the DFA for determining when to escape data while unparsing.
val resString = fieldReg.resultString.toString
- TLRegistersPool.returnToPool(fieldReg)
- TLRegistersPool.pool.finalCheck
+ state.dfaRegistersPool.returnToPool(fieldReg)
+ state.dfaRegistersPool.finalCheck
(resString, escapeOccurred)
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextPaddingParser.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextPaddingParser.scala
index 0941fc598..dd8cc4552 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextPaddingParser.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextPaddingParser.scala
@@ -24,6 +24,7 @@ import org.apache.daffodil.processors.DelimiterIterator
import org.apache.daffodil.io.DataInputStream
import scala.collection.mutable.ArrayBuffer
import org.apache.daffodil.io.FormatInfo
+import org.apache.daffodil.processors.parsers.PState
class TextPaddingParser(val padChar: Char,
override val context: TermRuntimeData)
@@ -34,9 +35,9 @@ class TextPaddingParser(val padChar: Char,
val paddingDFA = CreatePaddingDFA(padChar, context)
- def parse(finfo: FormatInfo, input: DataInputStream, delimIter: DelimiterIterator): Maybe[ParseResult] = {
+ def parse(finfo: PState, input: DataInputStream, delimIter: DelimiterIterator): Maybe[ParseResult] = {
- val paddingReg: Registers = TLRegistersPool.getFromPool("TextPaddingParser1")
+ val paddingReg: Registers = finfo.dfaRegistersPool.getFromPool("TextPaddingParser1")
paddingReg.reset(finfo, input, delimIter)
@@ -44,8 +45,8 @@ class TextPaddingParser(val padChar: Char,
val paddingValue = One(paddingReg.resultString.toString)
- TLRegistersPool.returnToPool(paddingReg)
- TLRegistersPool.pool.finalCheck
+ finfo.dfaRegistersPool.returnToPool(paddingReg)
+ finfo.dfaRegistersPool.finalCheck
One(new ParseResult(paddingValue, Nope, ArrayBuffer()))
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextParser.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextParser.scala
index 76f72bd58..be9517101 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextParser.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/TextParser.scala
@@ -23,15 +23,15 @@ import org.apache.daffodil.processors.TermRuntimeData
import org.apache.daffodil.processors.DelimiterIterator
import org.apache.daffodil.io.DataInputStream
import org.apache.daffodil.io.FormatInfo
+import org.apache.daffodil.processors.parsers.PState
-class TextParser(
- override val context: TermRuntimeData)
+class TextParser(override val context: TermRuntimeData)
extends DFAParser {
override lazy val name: String = "TextParser"
override lazy val info: String = "" // Nothing additional to add here
- def parse(finfo: FormatInfo, input: DataInputStream, delimIter: DelimiterIterator, isDelimRequired: Boolean): Maybe[ParseResult] = {
+ def parse(finfo: PState, input: DataInputStream, delimIter: DelimiterIterator, isDelimRequired: Boolean): Maybe[ParseResult] = {
val lmt = new LongestMatchTracker()
@@ -39,14 +39,14 @@ class TextParser(
delimIter.reset()
while (delimIter.hasNext()) {
val d = delimIter.next()
- val reg = TLRegistersPool.getFromPool("TextParser1")
+ val reg = finfo.dfaRegistersPool.getFromPool("TextParser1")
reg.reset(finfo, input, delimIter, m)
m = input.markPos
d.run(reg)
if (reg.status == StateKind.Succeeded) {
lmt.successfulMatch(reg.matchStartPos, reg.delimString, d, delimIter.currentIndex)
}
- TLRegistersPool.returnToPool(reg)
+ finfo.dfaRegistersPool.returnToPool(reg)
}
input.resetPos(m)
@@ -66,7 +66,7 @@ class TextParser(
}
}
- TLRegistersPool.pool.finalCheck
+ finfo.dfaRegistersPool.finalCheck
result
}
diff --git a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/RunnerFactory.scala b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/RunnerFactory.scala
index 36fb3b932..e5269a7b4 100644
--- a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/RunnerFactory.scala
+++ b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/RunnerFactory.scala
@@ -67,7 +67,20 @@ object Runner {
* A test or test suite can override this to specify more or different implementations
* that the test should pass for.
*/
- def defaultImplementationsDefaultDefault = Seq("daffodil")
+ def defaultImplementationsDefaultDefault = Seq("daffodil", "ibm")
+
+ /**
+ * By default we don't run Daffodil negative TDML tests against cross-testers.
+ * The error messages are simply too varied.
+ *
+ * Negative tests must fail, but error messages aren't compared.
+ */
+ def defaultShouldCrossTestNegativeTests = false
+
+ /**
+ * By default we don't cross test warning messages because they are too varied.
+ */
+ def defaultShouldCrossTestWarnings = false
}
diff --git a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala
index 834cf56d4..e4618198e 100644
--- a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala
+++ b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala
@@ -164,6 +164,9 @@ private[tdml] object DFDLTestSuite {
*
* defaultImplementationsDefault the implementations default for the test suite will be
* taken from this value if it is not specified on the testSuite itself.
+ *
+ * shouldCrossTestNegativeTests controls whether negative test error messages are compared
+ * or the tests are just run to determine that they fail.
*/
class DFDLTestSuite private[tdml] (val __nl: Null, // this extra arg allows us to make this primary constructor package private so we can deprecate the one generally used.
@@ -173,7 +176,9 @@ class DFDLTestSuite private[tdml] (val __nl: Null, // this extra arg allows us t
val compileAllTopLevel: Boolean,
val defaultRoundTripDefault: RoundTrip,
val defaultValidationDefault: String,
- val defaultImplementationsDefault: Seq[String])
+ val defaultImplementationsDefault: Seq[String],
+ val shouldCrossTestNegativeTests: Boolean,
+ val shouldCrossTestWarnings: Boolean)
extends Logging
with HasSetDebugger {
@@ -188,11 +193,15 @@ class DFDLTestSuite private[tdml] (val __nl: Null, // this extra arg allows us t
compileAllTopLevel: Boolean = false,
defaultRoundTripDefault: RoundTrip = Runner.defaultRoundTripDefaultDefault,
defaultValidationDefault: String = Runner.defaultValidationDefaultDefault,
- defaultImplementationsDefault: Seq[String] = Runner.defaultImplementationsDefaultDefault) =
+ defaultImplementationsDefault: Seq[String] = Runner.defaultImplementationsDefaultDefault,
+ shouldCrossTestNegativeTests: Boolean = Runner.defaultShouldCrossTestNegativeTests,
+ shouldCrossTestWarnings: Boolean = Runner.defaultShouldCrossTestWarnings) =
this(null, aNodeFileOrURL, validateTDMLFile, validateDFDLSchemas, compileAllTopLevel,
defaultRoundTripDefault,
defaultValidationDefault,
- defaultImplementationsDefault)
+ defaultImplementationsDefault,
+ shouldCrossTestNegativeTests,
+ shouldCrossTestWarnings)
if (!aNodeFileOrURL.isInstanceOf[scala.xml.Node])
System.err.println("Creating DFDL Test Suite for " + aNodeFileOrURL)
@@ -442,6 +451,10 @@ abstract class TestCase(testCaseXML: NodeSeq, val parent: DFDLTestSuite)
}
}
+ final def isCrossTest(implString: String) = implString != "daffodil"
+
+ final def isNegativeTest = optExpectedErrors.isDefined
+
lazy val tdmlDFDLProcessorFactory: AbstractTDMLDFDLProcessorFactory = {
import scala.language.reflectiveCalls
import scala.language.existentials
@@ -653,7 +666,10 @@ abstract class TestCase(testCaseXML: NodeSeq, val parent: DFDLTestSuite)
//
// Should we run the test?
//
- if (!implementationStrings.contains(impl.implementationName)) {
+ val implName = impl.implementationName
+ val istrings = implementationStrings
+ val useThisImpl = istrings.contains(implName)
+ if (!useThisImpl) {
//
// skip the test
//
@@ -706,6 +722,22 @@ abstract class TestCase(testCaseXML: NodeSeq, val parent: DFDLTestSuite)
}
}
+
+ protected def checkDiagnosticMessages(diagnostics: Seq[Throwable],
+ errors: ExpectedErrors,
+ optWarnings: Option[ExpectedWarnings],
+ implString: Option[String]) {
+ Assert.usage(this.isNegativeTest)
+
+ // check for any test-specified errors or warnings
+ if (!isCrossTest(implString.get) ||
+ parent.shouldCrossTestNegativeTests)
+ VerifyTestCase.verifyAllDiagnosticsFound(diagnostics, Some(errors), implString)
+
+ if (!isCrossTest(implString.get) ||
+ parent.shouldCrossTestWarnings)
+ VerifyTestCase.verifyAllDiagnosticsFound(diagnostics, optWarnings, implString)
+ }
}
case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
@@ -735,9 +767,9 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
}
}
- case (None, Some(_)) => {
+ case (None, Some(errors)) => {
compileResult.left.foreach { diags =>
- VerifyTestCase.verifyAllDiagnosticsFound(diags, optExpectedErrors, implString)
+ checkDiagnosticMessages(diags, errors, optExpectedWarnings, implString)
}
compileResult.right.foreach {
case (_, processor) =>
@@ -790,15 +822,11 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
(actual, diagnostics, isErr)
}
}
- // check for any test-specified warnings
- VerifyTestCase.verifyAllDiagnosticsFound(diagnostics, optWarnings, implString)
- if (isError) {
- // good we expected an error
- // check for any test-specified errors
- VerifyTestCase.verifyAllDiagnosticsFound(diagnostics, Some(errors), implString)
- } else {
+ if (!isError) {
throw TDMLException("Expected error. Didn't get one. Actual result was\n" + parseResult.getResult.toString, implString)
}
+
+ checkDiagnosticMessages(diagnostics, errors, optWarnings, implString)
}
/**
@@ -859,7 +887,9 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
}
val allDiags = processor.getDiagnostics ++ actual.getDiagnostics
- VerifyTestCase.verifyAllDiagnosticsFound(allDiags, optExpectedWarnings, implString)
+ if (!isCrossTest(implString.get) ||
+ parent.shouldCrossTestWarnings)
+ VerifyTestCase.verifyAllDiagnosticsFound(allDiags, optExpectedWarnings, implString)
}
/**
@@ -1067,9 +1097,7 @@ case class UnparserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
case (_, Some(errors)) => {
compileResult.left.foreach { diags =>
- VerifyTestCase.verifyAllDiagnosticsFound(diags, Some(errors), implString)
- // check warnings even if there are errors expected.
- VerifyTestCase.verifyAllDiagnosticsFound(diags, optWarnings, implString)
+ checkDiagnosticMessages(diags, errors, optWarnings, implString)
}
compileResult.right.foreach {
case (_, processor) =>
@@ -1123,7 +1151,9 @@ case class UnparserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
VerifyTestCase.verifyBinaryOrMixedData(expectedData, outStream, implString)
}
val allDiags = actual.getDiagnostics ++ processor.getDiagnostics
- VerifyTestCase.verifyAllDiagnosticsFound(allDiags, optWarnings, implString)
+ if (!isCrossTest(implString.get) ||
+ parent.shouldCrossTestWarnings)
+ VerifyTestCase.verifyAllDiagnosticsFound(allDiags, optWarnings, implString)
if (roundTrip eq OnePassRoundTrip) {
@@ -1151,7 +1181,9 @@ case class UnparserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
val xmlNode = parseActual.getResult
VerifyTestCase.verifyParserTestData(xmlNode, inputInfoset, implString)
- VerifyTestCase.verifyAllDiagnosticsFound(actual.getDiagnostics, optWarnings, implString)
+ if (!isCrossTest(implString.get) ||
+ parent.shouldCrossTestWarnings)
+ VerifyTestCase.verifyAllDiagnosticsFound(actual.getDiagnostics, optWarnings, implString)
(shouldValidate, expectsValidationError) match {
case (true, true) => {
@@ -1194,7 +1226,6 @@ case class UnparserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
case t: Throwable => toss(t, implString)
}
- // Verify that some partial output has shown up in the bytes.
val dataErrors = {
optExpectedData.flatMap { data =>
try {
@@ -1224,9 +1255,9 @@ case class UnparserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
}
}
- // check for any test-specified errors or warnings
- VerifyTestCase.verifyAllDiagnosticsFound(diagnostics, Some(errors), implString)
- VerifyTestCase.verifyAllDiagnosticsFound(diagnostics, optWarnings, implString)
+ if (diagnostics.isEmpty)
+ throw TDMLException("Unparser test expected error. Didn't get one.", implString)
+ checkDiagnosticMessages(diagnostics, errors, optWarnings, implString)
}
}
diff --git a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/nested-separator-delimited.tdml b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/nested-separator-delimited.tdml
index 5f87878db..6556962e4 100644
--- a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/nested-separator-delimited.tdml
+++ b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/nested-separator-delimited.tdml
@@ -21,7 +21,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"
xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns:ex="http://example.com" xmlns="http://example.com"
- defaultImplementations="ibm dfdl">
+ defaultImplementations="ibm daffodil">
<tdml:defineSchema name="baseline">
diff --git a/daffodil-test-ibm1/src/test/scala/org/apache/daffodil/TresysTests.scala b/daffodil-test-ibm1/src/test/scala/org/apache/daffodil/TresysTests.scala
index d5197dba7..1e48eae8a 100644
--- a/daffodil-test-ibm1/src/test/scala/org/apache/daffodil/TresysTests.scala
+++ b/daffodil-test-ibm1/src/test/scala/org/apache/daffodil/TresysTests.scala
@@ -109,10 +109,10 @@ class TresysTests {
// not found. Debug later.
// @Test def test_duplicateDefineFormatsOneSchema() { runnerMD.runOneTest("duplicateDefineFormatsOneSchema") }
- @Test def test_nested_separator_delimited_baseline() { runnerNSD.trace.runOneTest("baseline") }
+ @Test def test_nested_separator_delimited_baseline() { runnerNSD.runOneTest("baseline") }
// Fails in IBM DFDL - ambiguous separator/terminator not accepted.
- @Test def test_nested_separator_delimited_baseline_ibm() { runnerNSD.trace.runOneTest("baseline_ibm") }
+ @Test def test_nested_separator_delimited_baseline_ibm() { runnerNSD.runOneTest("baseline_ibm") }
@Test def test_nested_separator_delimited_basicNest() { runnerNSD.runOneTest("basicNest") }
// Fails infinite loop
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services