You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by sl...@apache.org on 2023/06/02 11:52:36 UTC
[daffodil] branch main updated: Remove warnings in serialized parser
This is an automated email from the ASF dual-hosted git repository.
slawrence pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/daffodil.git
The following commit(s) were added to refs/heads/main by this push:
new 4c17bfed4 Remove warnings in serialized parser
4c17bfed4 is described below
commit 4c17bfed49942588d0ea10981e3eb131b3cc71e5
Author: Mike McGann <mm...@owlcyberdefense.com>
AuthorDate: Wed May 31 07:56:52 2023 -0400
Remove warnings in serialized parser
Currently, when a DataProcessor is serialized using the save() function,
warnings generated during compilation are also serialized. But this
behavior is not needed, and can sometimes lead to excessive diagnostics
when reloading.
To avoid this, this moves the diagnostics member from the
SchemaSetRuntimeData to the DataProcessor, and clears that member before
serialization.
This also discovered an issue where the TDML runner was not correctly
checking for errors vs warnings and would accept messages of the wrong
type. This fixes that, which requires changes to some expected
errors/warnings.
That change also uncovered a bug where we only output a warning instead
of an error if the main schema is not a DFDL schema. This was caused by
an incorrect comparison in the "allSchemaFiles" logic, which also needed
to be a LV to handle the SDE correctly.
Also, the DaffodilTDMLDFDLProcessor has a getDiagnostics() function to
return compile time diagnostics. However, with this change, if a
processor is saved and reloaded by the TDML runner (which happens for
most tests) those diagnostics are lost. Because getDiagnostics may or
may not actually have diagnostics, this is an unreliable API and so is
removed. Instead, diagnostics should only be taken from the
CompileResult. For consistency, the isError function is also removed and
should only be determined based on the CompileResult.
Some asserts were added to make sure the CompileResult makes sense, and
dead code removed where it was impossible to have a bad processor.
DAFFODIL-2803
---
.../dsom/SchemaSetIncludesAndImportsMixins.scala | 6 +-
.../core/runtime1/SchemaSetRuntime1Mixin.scala | 5 +-
.../core/processor/TestSerialization.scala | 68 ++++++
.../runtime1/processors/DataProcessor.scala | 16 +-
.../runtime1/processors/SchemaSetRuntimeData.scala | 6 -
.../org/apache/daffodil/tdml/TDMLRunner.scala | 267 +++++++++++----------
.../tdml/processor/TDMLDFDLProcessor.scala | 4 -
.../tdml/DaffodilCTDMLDFDLProcessor.scala | 5 +-
.../processor/tdml/DaffodilTDMLDFDLProcessor.scala | 4 -
.../daffodil/processor/tdml/TestDaffodilC.scala | 1 -
.../org/apache/daffodil/layers/TestCheckDigit.tdml | 8 +-
.../general/testUnparserFileBuffering.tdml | 6 +-
.../section00/general/testUnparserGeneral.tdml | 6 +-
.../MissingAppinfoSource.dfdl.xsd | 3 +-
.../SchemaDefinitionErrors.tdml | 14 +-
.../daffodil/section07/assertions/assert.tdml | 8 +-
.../text_number_props/TextNumberProps.tdml | 2 +-
.../section23/dfdl_expressions/expressions.tdml | 5 +-
.../daffodil/unparser/parseUnparseModeTest.tdml | 2 +-
19 files changed, 257 insertions(+), 179 deletions(-)
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSetIncludesAndImportsMixins.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSetIncludesAndImportsMixins.scala
index ceb83011a..5dd5a220c 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSetIncludesAndImportsMixins.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSetIncludesAndImportsMixins.scala
@@ -61,10 +61,10 @@ trait SchemaSetIncludesAndImportsMixin { self: SchemaSet =>
allSchemaFiles.map { _.iiSchemaDocument }
}
- lazy val allSchemaFiles = {
+ lazy val allSchemaFiles = LV('allSchemaFiles) {
val fd = fakeXMLSchemaDocument // bootstrap
val sa = fd.seenAfter
- val first = sa.value.head._2
+ val first = sa.value.head._2.iiSchemaFile
val sfl = sa.value.flatMap {
case (_, ii) => {
val sf =
@@ -94,6 +94,6 @@ trait SchemaSetIncludesAndImportsMixin { self: SchemaSet =>
}
}.toList
sfl
- }
+ }.value
}
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/SchemaSetRuntime1Mixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/SchemaSetRuntime1Mixin.scala
index 224b419b4..534648bc9 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/SchemaSetRuntime1Mixin.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/runtime1/SchemaSetRuntime1Mixin.scala
@@ -79,12 +79,13 @@ trait SchemaSetRuntime1Mixin {
val p = if (!root.isError) parser else null
val u = if (!root.isError) unparser else null
val ssrd =
- new SchemaSetRuntimeData(p, u, this.diagnostics, rootERD, variableMap, typeCalcMap)
+ new SchemaSetRuntimeData(p, u, rootERD, variableMap, typeCalcMap)
if (root.numComponents > root.numUniqueComponents)
Logger.log.debug(
s"Compiler: component counts: unique ${root.numUniqueComponents}, actual ${root.numComponents}.",
)
- val dataProc = new DataProcessor(ssrd, tunable, variableMap.copy())
+ val dataProc =
+ new DataProcessor(ssrd, tunable, variableMap.copy(), diagnostics = this.diagnostics)
if (dataProc.isError) {} else {
Logger.log.debug(s"Parser = ${ssrd.parser.toString}.")
Logger.log.debug(s"Unparser = ${ssrd.unparser.toString}.")
diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/core/processor/TestSerialization.scala b/daffodil-core/src/test/scala/org/apache/daffodil/core/processor/TestSerialization.scala
new file mode 100644
index 000000000..dc43dddf4
--- /dev/null
+++ b/daffodil-core/src/test/scala/org/apache/daffodil/core/processor/TestSerialization.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.core.processor
+
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.nio.channels.Channels
+
+import org.apache.daffodil.core.compiler.Compiler
+
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class TestSerialization {
+
+ /**
+ * DAFFODIL-2803
+ *
+ * Check that warnings are not serialized when saving a parser.
+ */
+ @Test def test_stripWarnings() = {
+ val schema =
+ <schema
+ xmlns="http://www.w3.org/2001/XMLSchema"
+ xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"
+ xmlns:ex="http://example.com"
+ targetNamespace="http://example.com"
+ >
+ <include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/>
+ <annotation>
+ <!-- The invalid appinfo source generates a warning -->
+ <appinfo source="http://www.ogf.org/dfdl/WRONG">
+ <dfdl:format ref="ex:GeneralFormat"/>
+ </appinfo>
+ </annotation>
+ <element name="root" type="string" dfdl:lengthKind="explicit" dfdl:length="1"/>
+ </schema>
+
+ val factory = Compiler().compileNode(schema)
+ val processor = factory.onPath("/")
+ assertTrue(!processor.getDiagnostics.isEmpty)
+
+ val os = new ByteArrayOutputStream()
+ val output = Channels.newChannel(os)
+ processor.save(output)
+
+ val is = new ByteArrayInputStream(os.toByteArray)
+ val processor2 = Compiler().reload(is)
+
+ assertTrue(processor2.getDiagnostics.isEmpty)
+ }
+
+}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala
index 4f5d27a22..c32ab1671 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala
@@ -28,6 +28,7 @@ import java.nio.file.Files
import java.util.zip.GZIPOutputStream
import org.apache.daffodil.lib.Implicits._
+import org.apache.daffodil.lib.api.Diagnostic
import org.apache.daffodil.runtime1.layers.LayerExecutionException
object INoWarn4 {
@@ -137,6 +138,7 @@ class DataProcessor(
val validationMode: ValidationMode.Type = ValidationMode.Off,
protected val areDebugging: Boolean = false,
protected val optDebugger: Option[Debugger] = None,
+ protected val diagnostics: Seq[Diagnostic] = Seq.empty,
) extends DFDL.DataProcessor
with Serializable
with MultipleEventHandler {
@@ -171,7 +173,16 @@ class DataProcessor(
validationMode: ValidationMode.Type = validationMode,
areDebugging: Boolean = areDebugging,
optDebugger: Option[Debugger] = optDebugger,
- ) = new DataProcessor(ssrd, tunables, variableMap, validationMode, areDebugging, optDebugger)
+ diagnostics: Seq[Diagnostic] = diagnostics,
+ ) = new DataProcessor(
+ ssrd,
+ tunables,
+ variableMap,
+ validationMode,
+ areDebugging,
+ optDebugger,
+ diagnostics,
+ )
// This thread local state is used by the PState when it needs buffers for
// regex matching. This cannot be in PState because a PState does not last
@@ -261,7 +272,7 @@ class DataProcessor(
override def isError = false
- override def getDiagnostics = ssrd.diagnostics
+ override def getDiagnostics = diagnostics
override def newXMLReaderInstance: DFDL.DaffodilParseXMLReader = {
val xrdr = new DaffodilParseXMLReader(this)
@@ -295,6 +306,7 @@ class DataProcessor(
variableMap = ssrd.originalVariables, // reset to original variables defined in schema
validationMode =
ValidationMode.Off, // explicitly turn off, so restored processor won't be validating
+ diagnostics = Seq.empty, // don't save any warnings that were generated
)
try {
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/SchemaSetRuntimeData.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/SchemaSetRuntimeData.scala
index aaea41adf..ac0b7e3f8 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/SchemaSetRuntimeData.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/SchemaSetRuntimeData.scala
@@ -17,7 +17,6 @@
package org.apache.daffodil.runtime1.processors
-import org.apache.daffodil.lib.api.Diagnostic
import org.apache.daffodil.lib.exceptions.ThrowsSDE
import org.apache.daffodil.runtime1.processors.TypeCalculatorCompiler.TypeCalcMap
import org.apache.daffodil.runtime1.processors.parsers.Parser
@@ -26,11 +25,6 @@ import org.apache.daffodil.runtime1.processors.unparsers.Unparser
final class SchemaSetRuntimeData(
val parser: Parser,
val unparser: Unparser,
- /*
- * Memory of the compiler's warnings. If we save a processor, it's useful to be able
- * to have these warnings.
- */
- val diagnostics: Seq[Diagnostic],
val elementRuntimeData: ElementRuntimeData,
/*
* The original variables determined by the schema compiler.
diff --git a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala
index 0922c4ba3..007e93b1c 100644
--- a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala
+++ b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala
@@ -47,6 +47,7 @@ import org.apache.daffodil.lib.api.DaffodilConfig
import org.apache.daffodil.lib.api.DaffodilSchemaSource
import org.apache.daffodil.lib.api.DaffodilTunables
import org.apache.daffodil.lib.api.DataLocation
+import org.apache.daffodil.lib.api.Diagnostic
import org.apache.daffodil.lib.api.EmbeddedSchemaSource
import org.apache.daffodil.lib.api.TDMLImplementation
import org.apache.daffodil.lib.api.URISchemaSource
@@ -926,7 +927,7 @@ abstract class TestCase(testCaseXML: NodeSeq, val parent: DFDLTestSuite) {
}
protected def checkDiagnosticMessages(
- diagnostics: Seq[Throwable],
+ diagnostics: Seq[Diagnostic],
errors: ExpectedErrors,
optWarnings: Option[ExpectedWarnings],
implString: Option[String],
@@ -975,9 +976,9 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
(optExpectedInfoset, optExpectedErrors) match {
case (Some(_), None) => {
- compileResult.left.foreach { diags => throw TDMLException(diags, implString) }
- compileResult.right.foreach {
- case (_, proc) => {
+ compileResult match {
+ case Left(diags) => throw TDMLException(diags, implString)
+ case Right((diags, proc)) => {
processor = proc
runParseExpectSuccess(
dataToParse,
@@ -987,17 +988,17 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
validationMode,
roundTrip,
implString,
+ diags,
)
}
}
}
case (None, Some(errors)) => {
- compileResult.left.foreach { diags =>
- checkDiagnosticMessages(diags, errors, optExpectedWarnings, implString)
- }
- compileResult.right.foreach {
- case (_, proc) => {
+ compileResult match {
+ case Left(diags) =>
+ checkDiagnosticMessages(diags, errors, optExpectedWarnings, implString)
+ case Right((diags, proc)) => {
processor = proc
runParseExpectErrors(
dataToParse,
@@ -1007,6 +1008,7 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
optExpectedValidationErrors,
validationMode,
implString,
+ diags,
)
}
}
@@ -1024,6 +1026,7 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
optValidationErrors: Option[ExpectedValidationErrors],
validationMode: ValidationMode.Type,
implString: Option[String],
+ compileWarnings: Seq[Diagnostic],
): Unit = {
try {
@@ -1033,46 +1036,41 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
}
val (parseResult, diagnostics, isError) = {
- if (processor.isError) {
- val noParseResult: TDMLParseResult = null
- (noParseResult, processor.getDiagnostics, true)
- } else {
- val actual =
- try {
- processor.parse(dataToParse, lengthLimitInBits)
- } catch {
- case t: Throwable => toss(t, implString)
- }
+ val actual =
+ try {
+ processor.parse(dataToParse, lengthLimitInBits)
+ } catch {
+ case t: Throwable => toss(t, implString)
+ }
- // we should never need blobs if we're expecting an error even if we
- // don't get errors. So clean them up immediately
- actual.cleanUp()
+ // we should never need blobs if we're expecting an error even if we
+ // don't get errors. So clean them up immediately
+ actual.cleanUp()
- val isErr: Boolean =
- if (actual.isProcessingError) true
- else {
- //
- // didn't get an error.
- // If we're not at the end of data, synthesize an error for left-over-data
- //
- val loc: DataLocation = actual.currentLocation
-
- if (loc.bitPos1b >= 0 && loc.bitPos1b <= lengthLimitInBits) {
- val leftOverMsg =
- "Left over data. Consumed %s bit(s) with %s bit(s) remaining.".format(
- loc.bitPos1b - 1,
- lengthLimitInBits - (loc.bitPos1b - 1),
- )
- actual.addDiagnostic(new TDMLDiagnostic(leftOverMsg, implString))
- true
- } else {
- false
- }
+ val isErr: Boolean =
+ if (actual.isProcessingError) true
+ else {
+ //
+ // didn't get an error.
+ // If we're not at the end of data, synthesize an error for left-over-data
+ //
+ val loc: DataLocation = actual.currentLocation
+
+ if (loc.bitPos1b >= 0 && loc.bitPos1b <= lengthLimitInBits) {
+ val leftOverMsg =
+ "Left over data. Consumed %s bit(s) with %s bit(s) remaining.".format(
+ loc.bitPos1b - 1,
+ lengthLimitInBits - (loc.bitPos1b - 1),
+ )
+ actual.addDiagnostic(new TDMLDiagnostic(leftOverMsg, implString))
+ true
+ } else {
+ false
}
+ }
- val diagnostics = processor.getDiagnostics ++ actual.getDiagnostics
- (actual, diagnostics, isErr)
- }
+ val diagnostics = compileWarnings ++ actual.getDiagnostics
+ (actual, diagnostics, isErr)
}
if (!isError) {
toss(
@@ -1147,6 +1145,7 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
}
private def verifyParseResults(
+ compileWarnings: Seq[Diagnostic],
actual: TDMLParseResult,
testInfoset: Infoset,
implString: Option[String],
@@ -1176,7 +1175,7 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
case (false, false) => // Nothing to do here.
}
- val allDiags = processor.getDiagnostics ++ actual.getDiagnostics
+ val allDiags = compileWarnings ++ actual.getDiagnostics
if (
!isCrossTest(implString.get) ||
parent.shouldDoWarningComparisonOnCrossTests
@@ -1200,7 +1199,7 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
val unparseResult = processor.unparse(parseResult, outStream)
if (unparseResult.isProcessingError) {
- val diagObjs = processor.getDiagnostics ++ unparseResult.getDiagnostics
+ val diagObjs = unparseResult.getDiagnostics
if (diagObjs.length == 1) throw TDMLException(diagObjs.head, implString)
throw TDMLException(diagObjs, implString)
}
@@ -1266,16 +1265,12 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
validationMode: ValidationMode.Type,
roundTripArg: RoundTrip,
implString: Option[String],
+ compileWarnings: Seq[Diagnostic],
): Unit = {
val roundTrip =
roundTripArg // change to OnePassRoundTrip to force all parse tests to round trip (to see which fail to round trip)
- if (processor.isError) {
- val diagObjs = processor.getDiagnostics
- if (diagObjs.length == 1) throw diagObjs.head
- throw TDMLException(diagObjs, implString)
- }
processor = processor.withValidationMode(validationMode)
val firstParseTestData = IOUtils.toByteArray(dataToParse)
@@ -1286,7 +1281,7 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
roundTrip match {
case NoRoundTrip | OnePassRoundTrip => {
- verifyParseResults(firstParseResult, testInfoset, implString)
+ verifyParseResults(compileWarnings, firstParseResult, testInfoset, implString)
verifyLeftOverData(firstParseResult, lengthLimitInBits, implString)
}
case TwoPassRoundTrip => {
@@ -1321,7 +1316,7 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
//
// val isFirstParseInfosetMatching =
// try {
- // verifyParseResults(processor, firstParseResult, testInfoset, implString)
+ // verifyParseResults(compileWarnings, processor, firstParseResult, testInfoset, implString)
// verifyLeftOverData(firstParseResult, lengthLimitInBits, implString)
// true
// } catch {
@@ -1384,7 +1379,7 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
firstParseTestData,
testInfoset,
)
- verifyParseResults(actual, testInfoset, implString)
+ verifyParseResults(compileWarnings, actual, testInfoset, implString)
verifyLeftOverData(actual, reParseTestDataLength, implString)
// if it doesn't pass, it will throw out of here.
@@ -1423,7 +1418,7 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
//
// So we just verify normally here.
//
- verifyParseResults(secondParseResult, testInfoset, implString)
+ verifyParseResults(compileWarnings, secondParseResult, testInfoset, implString)
verifyLeftOverData(secondParseResult, reParseTestDataLength, implString)
//
// So now we do the third pass unparse and compare this output with the
@@ -1470,23 +1465,21 @@ case class UnparserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
(optExpectedData, optErrors) match {
case (Some(expectedData), None) => {
- compileResult.left.foreach { diags => throw TDMLException(diags, implString) }
- compileResult.right.foreach {
- case (warnings, proc) => {
+ compileResult match {
+ case Left(diags) => throw TDMLException(diags, implString)
+ case Right((diags, proc)) => {
processor = proc
- runUnparserExpectSuccess(expectedData, optWarnings, roundTrip, implString)
+ runUnparserExpectSuccess(expectedData, optWarnings, roundTrip, implString, diags)
}
}
}
case (_, Some(errors)) => {
- compileResult.left.foreach { diags =>
- checkDiagnosticMessages(diags, errors, optWarnings, implString)
- }
- compileResult.right.foreach {
- case (_, proc) => {
+ compileResult match {
+ case Left(diags) => checkDiagnosticMessages(diags, errors, optWarnings, implString)
+ case Right((diags, proc)) => {
processor = proc
- runUnparserExpectErrors(optExpectedData, errors, optWarnings, implString)
+ runUnparserExpectErrors(optExpectedData, errors, optWarnings, implString, diags)
}
}
}
@@ -1500,6 +1493,7 @@ case class UnparserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
optWarnings: Option[ExpectedWarnings],
roundTrip: RoundTrip,
implString: Option[String],
+ compileWarnings: Seq[Diagnostic],
): Unit = {
Assert.usage(roundTrip ne TwoPassRoundTrip) // not supported for unparser test cases.
@@ -1542,7 +1536,7 @@ case class UnparserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
// we will need to treat as Hex bytes as well.
VerifyTestCase.verifyBinaryOrMixedData(expectedData, outStream, implString)
}
- val allDiags = actual.getDiagnostics ++ processor.getDiagnostics
+ val allDiags = actual.getDiagnostics ++ compileWarnings
if (
!isCrossTest(implString.get) ||
parent.shouldDoWarningComparisonOnCrossTests
@@ -1631,6 +1625,7 @@ case class UnparserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
errors: ExpectedErrors,
optWarnings: Option[ExpectedWarnings],
implString: Option[String],
+ compileWarnings: Seq[Diagnostic],
): Unit = {
try {
@@ -1639,59 +1634,53 @@ case class UnparserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
case e: Exception => throw TDMLException(e, implString)
}
- val diagnostics = {
- if (processor.isError)
- processor.getDiagnostics
- else {
- val outStream = new java.io.ByteArrayOutputStream()
- val infosetXML = {
- if (optInputInfoset.isEmpty)
- throw TDMLException(
- "No infoset specified, but one is required to run the test.",
- implString,
- )
- inputInfoset.dfdlInfoset.contents
- }
- val actual =
- try {
- processor.unparse(infosetXML, outStream)
- } catch {
- case t: Throwable => toss(t, implString)
- }
+ val outStream = new java.io.ByteArrayOutputStream()
+ val infosetXML = {
+ if (optInputInfoset.isEmpty)
+ throw TDMLException(
+ "No infoset specified, but one is required to run the test.",
+ implString,
+ )
+ inputInfoset.dfdlInfoset.contents
+ }
+ val actual =
+ try {
+ processor.unparse(infosetXML, outStream)
+ } catch {
+ case t: Throwable => toss(t, implString)
+ }
- val dataErrors = {
- optExpectedData.flatMap { data =>
- try {
- if (actual.isScannable) {
- // all textual in one encoding, so we can do display of results
- // in terms of text so the user can see what is going on.
- VerifyTestCase.verifyTextData(data, outStream, actual.encodingName, implString)
- } else {
- // data is not all textual, or in mixture of encodings
- // So while we can still use the encoding as a heuristic,
- // we will need to treat as Hex bytes as well.
- VerifyTestCase.verifyBinaryOrMixedData(data, outStream, implString)
- }
- None
- } catch {
- //
- // verifyData throws TDMLExceptions if the data doesn't match
- // In order for negative tests to look for these errors
- // we need to capture them and treat like regular diagnostics.
- //
- case x: TDMLException =>
- Some(x)
- }
+ val dataErrors = {
+ optExpectedData.flatMap { data =>
+ try {
+ if (actual.isScannable) {
+ // all textual in one encoding, so we can do display of results
+ // in terms of text so the user can see what is going on.
+ VerifyTestCase.verifyTextData(data, outStream, actual.encodingName, implString)
+ } else {
+ // data is not all textual, or in mixture of encodings
+ // So while we can still use the encoding as a heuristic,
+ // we will need to treat as Hex bytes as well.
+ VerifyTestCase.verifyBinaryOrMixedData(data, outStream, implString)
}
+ None
+ } catch {
+ //
+ // verifyData throws TDMLExceptions if the data doesn't match
+ // In order for negative tests to look for these errors
+ // we need to capture them and treat like regular diagnostics.
+ //
+ case x: TDMLException =>
+ Some(new TDMLDiagnostic(x.getMessage, implString))
}
-
- // Done with the unparse results, safe to clean up any temporary files
- actual.cleanUp()
-
- processor.getDiagnostics ++ actual.getDiagnostics ++ dataErrors
}
}
+ // Done with the unparse results, safe to clean up any temporary files
+ actual.cleanUp()
+
+ val diagnostics = compileWarnings ++ actual.getDiagnostics ++ dataErrors
+
if (diagnostics.isEmpty)
throw TDMLException("Unparser test expected error. Didn't get one.", implString)
checkDiagnosticMessages(diagnostics, errors, optWarnings, implString)
@@ -1753,7 +1742,7 @@ object VerifyTestCase {
}
def verifyAllDiagnosticsFound(
- actualDiags: Seq[Throwable],
+ actualDiags: Seq[Diagnostic],
expectedDiags: Option[ErrorWarningBase],
implString: Option[String],
) = {
@@ -1781,19 +1770,24 @@ object VerifyTestCase {
}
// must find each expected warning message within some actual warning message.
- expectedDiagMsgs.foreach { expected =>
+ expectedDiags.foreach { expectedDiag =>
{
- val wasFound = actualDiagMsgs.exists { actual =>
- actual.toLowerCase.contains(expected.toLowerCase)
- }
- if (!wasFound) {
- throw TDMLException(
- """Did not find diagnostic message """" +
- expected +
- """" in any of the actual diagnostic messages: """ + "\n" +
- actualDiagMsgs.mkString("\n"),
- implString,
- )
+ expectedDiag.messages.foreach { expectedMsg =>
+ {
+ val wasFound = actualDiags.exists { actualDiag =>
+ actualDiag.isError == expectedDiag.isError &&
+ actualDiag.getMessage.toLowerCase.contains(expectedMsg.toLowerCase)
+ }
+ if (!wasFound) {
+ throw TDMLException(
+ s"""Did not find diagnostic ${expectedDiag.diagnosticType} message """" +
+ expectedMsg +
+ """" in any of the actual diagnostic messages: """ + "\n" +
+ actualDiagMsgs.mkString("\n"),
+ implString,
+ )
+ }
+ }
}
}
}
@@ -2707,6 +2701,9 @@ case class DFDLInfoset(di: Node, parent: Infoset) {
abstract class ErrorWarningBase(n: NodeSeq, parent: TestCase) {
lazy val matchAttrib = (n \ "@match").text
+ def isError: Boolean
+ def diagnosticType: String
+
protected def diagnosticNodes: Seq[Node]
lazy val messages = diagnosticNodes.map {
@@ -2720,6 +2717,8 @@ case class ExpectedErrors(node: NodeSeq, parent: TestCase)
extends ErrorWarningBase(node, parent) {
val diagnosticNodes = node \\ "error"
+ override def isError = true
+ override def diagnosticType = "error"
}
@@ -2727,6 +2726,8 @@ case class ExpectedWarnings(node: NodeSeq, parent: TestCase)
extends ErrorWarningBase(node, parent) {
val diagnosticNodes = node \\ "warning"
+ override def isError = false
+ override def diagnosticType = "warning"
}
@@ -2734,6 +2735,8 @@ case class ExpectedValidationErrors(node: NodeSeq, parent: TestCase)
extends ErrorWarningBase(node, parent) {
val diagnosticNodes = node \\ "error"
+ override def isError = true
+ override def diagnosticType = "validation error"
}
@@ -2945,6 +2948,16 @@ case class TDMLCompileResultCache(entryExpireDurationSeconds: Option[Long]) {
key.tunables,
)
cache += (key -> TDMLCompileResultCacheValue(compileResult, None))
+ compileResult match {
+ case Left(diags) => {
+ // a Left must have at least one error diagnostic if we don't get a processor
+ Assert.invariant(diags.exists(_.isError))
+ }
+ case Right((diags, _)) => {
+ // if a Right has diags, they must all be warnings
+ Assert.invariant(diags.forall(!_.isError))
+ }
+ }
compileResult
}
}
diff --git a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/processor/TDMLDFDLProcessor.scala b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/processor/TDMLDFDLProcessor.scala
index aa8aa071e..04baa019f 100644
--- a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/processor/TDMLDFDLProcessor.scala
+++ b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/processor/TDMLDFDLProcessor.scala
@@ -81,10 +81,6 @@ trait TDMLDFDLProcessor {
def withExternalDFDLVariables(externalVarBindings: Seq[Binding]): R
- def isError: Boolean
-
- def getDiagnostics: Seq[Diagnostic]
-
def parse(is: java.io.InputStream, lengthLimitInBits: Long): TDMLParseResult
def unparse(infosetXML: scala.xml.Node, outStream: java.io.OutputStream): TDMLUnparseResult
diff --git a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/DaffodilCTDMLDFDLProcessor.scala b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/DaffodilCTDMLDFDLProcessor.scala
index 83562c1fa..772f0e34b 100644
--- a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/DaffodilCTDMLDFDLProcessor.scala
+++ b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/DaffodilCTDMLDFDLProcessor.scala
@@ -127,9 +127,6 @@ final class DaffodilCTDMLDFDLProcessor(executable: os.Path) extends TDMLDFDLProc
override def withDebugger(db: AnyRef): R = this
override def withValidationMode(validationMode: ValidationMode.Type): R = this
override def withExternalDFDLVariables(externalVarBindings: Seq[Binding]): R = this
- // We return errors and diagnostics in TDMLParseResult and TDMLUnparseResult
- override def isError: Boolean = false
- override def getDiagnostics: Seq[Diagnostic] = Seq.empty
// Parses the input stream to an infoset and returns a TDMLParseResult
// containing the infoset with any errors and diagnostics.
@@ -268,7 +265,7 @@ final case class DaffodilCTDMLMessages(messages: String)
extends SchemaDefinitionDiagnosticBase(Nope, Nope, None, Nope, Maybe(messages)) {
override def isValidation: Boolean = true
- override def isError: Boolean = false
+ override def isError: Boolean = true
override protected def modeName: String = TDMLImplementation.DaffodilC.toString
}
diff --git a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/DaffodilTDMLDFDLProcessor.scala b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/DaffodilTDMLDFDLProcessor.scala
index 6bf60ebfc..d4189b8f7 100644
--- a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/DaffodilTDMLDFDLProcessor.scala
+++ b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/DaffodilTDMLDFDLProcessor.scala
@@ -205,10 +205,6 @@ class DaffodilTDMLDFDLProcessor private (private var dp: DataProcessor)
): DaffodilTDMLDFDLProcessor =
copy(dp = dp.withExternalVariables(externalVarBindings))
- override def isError: Boolean = dp.isError
-
- override def getDiagnostics: Seq[Diagnostic] = dp.getDiagnostics
-
def parse(uri: java.net.URI, lengthLimitInBits: Long): TDMLParseResult = {
val url = uri.toURL
val dpInputStream = url.openStream()
diff --git a/daffodil-tdml-processor/src/test/scala/org/apache/daffodil/processor/tdml/TestDaffodilC.scala b/daffodil-tdml-processor/src/test/scala/org/apache/daffodil/processor/tdml/TestDaffodilC.scala
index 45e845cda..53cf17f88 100644
--- a/daffodil-tdml-processor/src/test/scala/org/apache/daffodil/processor/tdml/TestDaffodilC.scala
+++ b/daffodil-tdml-processor/src/test/scala/org/apache/daffodil/processor/tdml/TestDaffodilC.scala
@@ -219,7 +219,6 @@ class TestDaffodilC {
case Left(diagnostics) => fail(s"getProcessor failed: ${diagnostics.mkString}")
case Right((diagnostics, tdmlDFDLProcessor)) =>
assert(diagnostics.isEmpty)
- assert(!tdmlDFDLProcessor.isError)
assert(tdmlDFDLProcessor.isInstanceOf[DaffodilCTDMLDFDLProcessor])
}
}
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/layers/TestCheckDigit.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/layers/TestCheckDigit.tdml
index 005a36365..edd085e7c 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/layers/TestCheckDigit.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/layers/TestCheckDigit.tdml
@@ -63,10 +63,10 @@
<computedCheckDigit>1</computedCheckDigit>
</ex:r>
</dfdlInfoset></infoset>
- <warnings>
- <warning>Incorrect check digit</warning>
- <warning>7</warning>
- </warnings>
+ <validationErrors>
+ <error>Incorrect check digit</error>
+ <error>7</error>
+ </validationErrors>
</parserTestCase>
<unparserTestCase name="test_checkDigit_01u" root="r" model="checkDigit.dfdl.xsd"
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/testUnparserFileBuffering.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/testUnparserFileBuffering.tdml
index 128a7669f..649c0702b 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/testUnparserFileBuffering.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/testUnparserFileBuffering.tdml
@@ -256,7 +256,7 @@
<tdml:document>incorrect</tdml:document>
<tdml:errors>
- <tdml:error>TDMLException</tdml:error>
+ <tdml:error>TDML Error</tdml:error>
<tdml:error>output data length</tdml:error>
<tdml:error>length 1</tdml:error>
<tdml:error>'0'</tdml:error>
@@ -285,7 +285,7 @@
<tdml:document>Q</tdml:document>
<tdml:errors>
- <tdml:error>TDMLException</tdml:error>
+ <tdml:error>TDML Error</tdml:error>
<tdml:error>data differs</tdml:error>
<tdml:error>Expected 'Q'</tdml:error>
<tdml:error>Actual was 'X'</tdml:error>
@@ -367,7 +367,7 @@
<tdml:document>Xtra</tdml:document>
<tdml:errors>
- <tdml:error>TDMLException</tdml:error>
+ <tdml:error>TDML Error</tdml:error>
<tdml:error>output data length 1</tdml:error>
<tdml:error>'X'</tdml:error>
<tdml:error>doesn't match expected length 4</tdml:error>
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/testUnparserGeneral.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/testUnparserGeneral.tdml
index 36fac816d..e194b57e0 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/testUnparserGeneral.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/testUnparserGeneral.tdml
@@ -251,7 +251,7 @@
<tdml:document>incorrect</tdml:document>
<tdml:errors>
- <tdml:error>TDMLException</tdml:error>
+ <tdml:error>TDML Error</tdml:error>
<tdml:error>output data length</tdml:error>
<tdml:error>length 1</tdml:error>
<tdml:error>'0'</tdml:error>
@@ -279,7 +279,7 @@
<tdml:document>Q</tdml:document>
<tdml:errors>
- <tdml:error>TDMLException</tdml:error>
+ <tdml:error>TDML Error</tdml:error>
<tdml:error>data differs</tdml:error>
<tdml:error>Expected 'Q'</tdml:error>
<tdml:error>Actual was 'X'</tdml:error>
@@ -358,7 +358,7 @@
<tdml:document>Xtra</tdml:document>
<tdml:errors>
- <tdml:error>TDMLException</tdml:error>
+ <tdml:error>TDML Error</tdml:error>
<tdml:error>output data length 1</tdml:error>
<tdml:error>'X'</tdml:error>
<tdml:error>doesn't match expected length 4</tdml:error>
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section02/schema_definition_errors/MissingAppinfoSource.dfdl.xsd b/daffodil-test/src/test/resources/org/apache/daffodil/section02/schema_definition_errors/MissingAppinfoSource.dfdl.xsd
index 7f576e36a..802e2a765 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/section02/schema_definition_errors/MissingAppinfoSource.dfdl.xsd
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section02/schema_definition_errors/MissingAppinfoSource.dfdl.xsd
@@ -21,9 +21,10 @@
xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"
xmlns:fn="http://www.w3.org/2005/xpath-functions"
>
+ <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/>
<xs:annotation>
<xs:appinfo source="http://www.ogf.org/dfdl/">
- <dfdl:format
+ <dfdl:format ref="GeneralFormat"
terminator="" leadingSkip='0' textTrimKind="none" initiatedContent="no"
alignment="implicit" alignmentUnits="bits" trailingSkip="0" ignoreCase="no"
separatorPosition="infix" occursCountKind="implicit"
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section02/schema_definition_errors/SchemaDefinitionErrors.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section02/schema_definition_errors/SchemaDefinitionErrors.tdml
index b58ff0111..ef3d319b2 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/section02/schema_definition_errors/SchemaDefinitionErrors.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section02/schema_definition_errors/SchemaDefinitionErrors.tdml
@@ -182,12 +182,14 @@
<tdml:parserTestCase name="missing_appinfo_source" root="elem"
model="MissingAppinfoSource.dfdl.xsd"
description="">
- <tdml:document><![CDATA[test]]></tdml:document>
-
- <tdml:errors>
- <tdml:error>Schema Definition Warning</tdml:error>
- <tdml:error>xs:appinfo without source attribute</tdml:error>
- </tdml:errors>
+ <tdml:document/>
+ <tdml:infoset>
+ <tdml:dfdlInfoset><elem/></tdml:dfdlInfoset>
+ </tdml:infoset>
+ <tdml:warnings>
+ <tdml:warning>Schema Definition Warning</tdml:warning>
+ <tdml:warning>xs:appinfo without source attribute</tdml:warning>
+ </tdml:warnings>
</tdml:parserTestCase>
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section07/assertions/assert.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section07/assertions/assert.tdml
index a44320112..cc6c6f78d 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/section07/assertions/assert.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section07/assertions/assert.tdml
@@ -660,10 +660,10 @@
root="e6r" model="s1" roundTrip="false"
description="Section 7 - assert pattern fail multiple assert - DFDL-7-053R">
<tdml:document>43,</tdml:document>
- <tdml:warnings>
- <tdml:warning>Assert</tdml:warning>
- <tdml:warning>pattern</tdml:warning>
- </tdml:warnings>
+ <tdml:validationErrors>
+ <tdml:error>Assert</tdml:error>
+ <tdml:error>pattern</tdml:error>
+ </tdml:validationErrors>
<tdml:infoset>
<tdml:dfdlInfoset>
<e6r>43</e6r>
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section13/text_number_props/TextNumberProps.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section13/text_number_props/TextNumberProps.tdml
index f045cc63c..cb4fda7ed 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/section13/text_number_props/TextNumberProps.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section13/text_number_props/TextNumberProps.tdml
@@ -3761,7 +3761,7 @@
<tdml:errors>
<tdml:error>Schema Definition Error</tdml:error>
<tdml:error>textStandardExponentRep</tdml:error>
- <tdml:error>cannot</tdml:error>
+ <tdml:error>disallowed</tdml:error>
<tdml:error>contain</tdml:error>
<tdml:error>ES</tdml:error>
</tdml:errors>
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section23/dfdl_expressions/expressions.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section23/dfdl_expressions/expressions.tdml
index 00f6858b1..eed3000de 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/section23/dfdl_expressions/expressions.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section23/dfdl_expressions/expressions.tdml
@@ -7253,8 +7253,7 @@
<tdml:document>1,2,3</tdml:document>
<tdml:errors>
<tdml:error>Schema Definition Error</tdml:error>
- <tdml:error>Statically ambiguous or query-style paths not supported</tdml:error>
- <tdml:error>ex:a</tdml:error>
+ <tdml:error>Path step 'ex:{http://example.com}a' ambiguous</tdml:error>
</tdml:errors>
</tdml:parserTestCase>
@@ -7271,7 +7270,7 @@
<tdml:document>1,2,3</tdml:document>
<tdml:errors>
<tdml:error>Schema Definition Error</tdml:error>
- <tdml:error>Statically ambiguous or query-style paths not supported</tdml:error>
+ <tdml:error>Subset Indexing is only allowed on arrays</tdml:error>
<tdml:error>ex:a</tdml:error>
</tdml:errors>
</tdml:parserTestCase>
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/unparser/parseUnparseModeTest.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/unparser/parseUnparseModeTest.tdml
index a7d31e1d7..75240cd88 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/unparser/parseUnparseModeTest.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/unparser/parseUnparseModeTest.tdml
@@ -55,7 +55,7 @@
</tdml:dfdlInfoset>
</tdml:infoset>
<tdml:errors>
- <tdml:error>Schema Definition Warning</tdml:error>
+ <tdml:error>Schema Definition Error</tdml:error>
<tdml:error>UPA Violation</tdml:error>
<tdml:error>multiple choice</tdml:error>
</tdml:errors>