You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by ja...@apache.org on 2019/04/24 14:20:43 UTC
[incubator-daffodil] branch master updated: Escape all delimiters
when unparsing
This is an automated email from the ASF dual-hosted git repository.
jadams pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git
The following commit(s) were added to refs/heads/master by this push:
new 6101cdb Escape all delimiters when unparsing
6101cdb is described below
commit 6101cdbbc9876d3b18b84d0a6ddfd528a3ed0386
Author: Josh Adams <ja...@tresys.com>
AuthorDate: Mon Apr 22 13:34:57 2019 -0400
Escape all delimiters when unparsing
Previously we were only escaping the first delimiter in the list of
delimiters specified by initiator, terminator and separator. Now we will
escape all delimiters specified in these lists.
DAFFODIL-2089
---
.../unparsers/ChoiceAndOtherVariousUnparsers.scala | 7 +++---
.../processors/unparsers/DelimiterUnparsers.scala | 6 ++---
.../processors/DelimiterStackUnparseNode.scala | 18 +++++++-------
.../apache/daffodil/processors/EvDelimiters.scala | 12 +++++-----
.../processors/dfa/CreateDelimiterDFA.scala | 9 +++++++
.../escapeScheme/escapeSchemeUnparse.tdml | 28 +++++++++++++++++++++-
.../escapeScheme/TestEscapeSchemeUnparse.scala | 1 +
7 files changed, 58 insertions(+), 23 deletions(-)
diff --git a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/ChoiceAndOtherVariousUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/ChoiceAndOtherVariousUnparsers.scala
index 29b8dd2..0a9f432 100644
--- a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/ChoiceAndOtherVariousUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/ChoiceAndOtherVariousUnparsers.scala
@@ -20,6 +20,7 @@ package org.apache.daffodil.processors.unparsers
import org.apache.daffodil.processors._
import org.apache.daffodil.infoset._
import org.apache.daffodil.processors.RuntimeData
+import org.apache.daffodil.processors.dfa.DFADelimiter
import org.apache.daffodil.util.Maybe._
import org.apache.daffodil.util.Maybe
import org.apache.daffodil.util.Maybe._
@@ -117,9 +118,9 @@ class DelimiterStackUnparser(
def unparse(state: UState): Unit = {
// Evaluate Delimiters
- val init = if (initiatorOpt.isDefined) Maybe.toMaybe(initiatorOpt.get.evaluate(state)) else Nope
- val sep = if (separatorOpt.isDefined) Maybe.toMaybe(separatorOpt.get.evaluate(state)) else Nope
- val term = if (terminatorOpt.isDefined) Maybe.toMaybe(terminatorOpt.get.evaluate(state)) else Nope
+ val init = if (initiatorOpt.isDefined) initiatorOpt.get.evaluate(state) else Array[DFADelimiter]()
+ val sep = if (separatorOpt.isDefined) separatorOpt.get.evaluate(state) else Array[DFADelimiter]()
+ val term = if (terminatorOpt.isDefined) terminatorOpt.get.evaluate(state) else Array[DFADelimiter]()
val node = DelimiterStackUnparseNode(init, sep, term)
diff --git a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/DelimiterUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/DelimiterUnparsers.scala
index 7581605..9da7ce8 100644
--- a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/DelimiterUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/DelimiterUnparsers.scala
@@ -50,15 +50,15 @@ class DelimiterTextUnparser(override val context: TermRuntimeData, delimiterType
val localDelimNode = state.localDelimiters
- val delimDFAOpt = {
+ val delimDFAs = {
if (delimiterType == DelimiterTextType.Initiator) localDelimNode.initiator
else if (delimiterType == DelimiterTextType.Separator) localDelimNode.separator
else localDelimNode.terminator
}
- if (!delimDFAOpt.isDefined) Assert.invariantFailed("Expected a delimiter of type " + delimiterType + " on the stack, but was not found.")
+ if (delimDFAs.isEmpty) Assert.invariantFailed("Expected a delimiter of type " + delimiterType + " on the stack, but was not found.")
- val delimDFA = delimDFAOpt.get
+ val delimDFA = delimDFAs(0)
try {
val valueString = delimDFA.unparseValue
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DelimiterStackUnparseNode.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DelimiterStackUnparseNode.scala
index a182105..3131cae 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DelimiterStackUnparseNode.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DelimiterStackUnparseNode.scala
@@ -17,30 +17,28 @@
package org.apache.daffodil.processors
-import org.apache.daffodil.util.Maybe
-import org.apache.daffodil.util.Maybe.Nope
import org.apache.daffodil.processors.dfa.DFADelimiter
object EmptyDelimiterStackUnparseNode {
- val node = new DelimiterStackUnparseNode(Nope, Nope, Nope)
+ val node = new DelimiterStackUnparseNode(Array(), Array(), Array())
def apply() = node
}
object DelimiterStackUnparseNode {
def apply(
- initiator: Maybe[DFADelimiter],
- separator: Maybe[DFADelimiter],
- terminator: Maybe[DFADelimiter]): DelimiterStackUnparseNode = {
- if (!initiator.isDefined && !terminator.isDefined && !separator.isDefined) EmptyDelimiterStackUnparseNode()
+ initiator: Array[DFADelimiter],
+ separator: Array[DFADelimiter],
+ terminator: Array[DFADelimiter]): DelimiterStackUnparseNode = {
+ if (initiator.isEmpty && terminator.isEmpty && separator.isEmpty) EmptyDelimiterStackUnparseNode()
else new DelimiterStackUnparseNode(initiator, separator, terminator)
}
}
class DelimiterStackUnparseNode(
- val initiator: Maybe[DFADelimiter],
- val separator: Maybe[DFADelimiter],
- val terminator: Maybe[DFADelimiter]) {
+ val initiator: Array[DFADelimiter],
+ val separator: Array[DFADelimiter],
+ val terminator: Array[DFADelimiter]) {
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/EvDelimiters.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/EvDelimiters.scala
index 398d881..c55688b 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/EvDelimiters.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/EvDelimiters.scala
@@ -74,23 +74,23 @@ abstract class DelimiterParseEv(delimType: DelimiterTextType.Type, override val
}
abstract class DelimiterUnparseEv(delimType: DelimiterTextType.Type, override val expr: CompiledExpression[String], outputNewLine: OutputNewLineEv, override val trd: TermRuntimeData)
- extends Evaluatable[Option[DFADelimiter]](trd)
- with InfosetCachedEvaluatable[Option[DFADelimiter]]
- with DelimiterEvMixin[Option[DFADelimiter]] {
+ extends Evaluatable[Array[DFADelimiter]](trd)
+ with InfosetCachedEvaluatable[Array[DFADelimiter]]
+ with DelimiterEvMixin[Array[DFADelimiter]] {
override lazy val runtimeDependencies = Seq(outputNewLine)
- override protected def compute(state: ParseOrUnparseState): Option[DFADelimiter] = {
+ override protected def compute(state: ParseOrUnparseState): Array[DFADelimiter] = {
if (state.isInstanceOf[PState]) {
Assert.invariantFailed("State was PState in Unparser Evaluatable")
}
val converterResult = evalAndConvert(state)
if (converterResult.length == 1 && converterResult(0) == "") {
- None
+ Array()
} else {
val onl = outputNewLine.evaluate(state)
- Some(CreateDelimiterDFA(delimType, trd, converterResult(0), onl))
+ CreateDelimiterDFA(delimType, trd, converterResult, onl)
}
}
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/CreateDelimiterDFA.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/CreateDelimiterDFA.scala
index 20d3556..ceb3a97 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/CreateDelimiterDFA.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/dfa/CreateDelimiterDFA.scala
@@ -28,6 +28,7 @@ import org.apache.daffodil.processors.WSPPlusDelim
import org.apache.daffodil.processors.WSPStarDelim
import org.apache.daffodil.processors.RuntimeData
import org.apache.daffodil.processors.parsers.DelimiterTextType
+import org.apache.daffodil.exceptions.Assert
object CreateDelimiterDFA {
@@ -82,6 +83,14 @@ object CreateDelimiterDFA {
/**
* Converts a Seq of String to a Seq of
+ * DFA's representing each String with outputNewLine.
+ */
+ def apply(delimType: DelimiterTextType.Type, rd: RuntimeData, delimiters: Seq[String], outputNewLine: String): Array[DFADelimiter] = {
+ delimiters.map(d => apply(delimType, rd, d, outputNewLine)).toArray
+ }
+
+ /**
+ * Converts a Seq of String to a Seq of
* DFA's representing each String.
*/
def apply(delimType: DelimiterTextType.Type, rd: RuntimeData, delimiters: Seq[String], ignoreCase: Boolean): Array[DFADelimiter] = {
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section07/escapeScheme/escapeSchemeUnparse.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section07/escapeScheme/escapeSchemeUnparse.tdml
index f97cfce..24ec3a6 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/section07/escapeScheme/escapeSchemeUnparse.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section07/escapeScheme/escapeSchemeUnparse.tdml
@@ -134,6 +134,14 @@
</xs:complexType>
</xs:element>
+ <xs:element name="e10">
+ <xs:complexType>
+ <xs:sequence dfdl:separator="| ; ," >
+ <xs:element name="s1" type="xs:string" dfdl:lengthKind="delimited" dfdl:escapeSchemeRef="pound" minOccurs="0" maxOccurs="6" dfdl:occursCountKind="parsed"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
</tdml:defineSchema>
<!--
@@ -397,7 +405,25 @@
<tdml:error>exactly 1 character</tdml:error>
</tdml:errors>
</tdml:unparserTestCase>
-
+
+<!--
+ Test Name: unparseDelimitedEscapedString22
+ Schema: delimitedStringsEscapeScheme
+ Purpose: This test demonstrates unparsing a sequence of delimited strings with escape schemes
+-->
+ <tdml:unparserTestCase name="unparseDelimitedEscapedString22" model="delimitedStringsEscapeScheme" root="e10" roundTrip="true">
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e10 xmlns:ex="http://example.com">
+ <ex:s1>one,two</ex:s1>
+ <ex:s1>three|four</ex:s1>
+ <ex:s1>five;six</ex:s1>
+ </ex:e10>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ <tdml:document>one#,two|three#|four|five#;six</tdml:document>
+ </tdml:unparserTestCase>
+
<tdml:defineSchema name="foo">
<xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/>
<dfdl:format ref="ex:GeneralFormat" encoding="ascii" lengthUnits="bytes" outputNewLine="%CR;%LF;"/>
diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section07/escapeScheme/TestEscapeSchemeUnparse.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section07/escapeScheme/TestEscapeSchemeUnparse.scala
index 318bf09..8a643be 100644
--- a/daffodil-test/src/test/scala/org/apache/daffodil/section07/escapeScheme/TestEscapeSchemeUnparse.scala
+++ b/daffodil-test/src/test/scala/org/apache/daffodil/section07/escapeScheme/TestEscapeSchemeUnparse.scala
@@ -66,6 +66,7 @@ class TestEscapeSchemeUnparse {
@Test def test_unparseDelimitedEscapedString19() { runner.runOneTest("unparseDelimitedEscapedString19") }
@Test def test_unparseDelimitedEscapedString20() { runner.runOneTest("unparseDelimitedEscapedString20") }
@Test def test_unparseDelimitedEscapedString21() { runner.runOneTest("unparseDelimitedEscapedString21") }
+ @Test def test_unparseDelimitedEscapedString22() { runner.runOneTest("unparseDelimitedEscapedString22") }
@Test def test_parseDelimitedEscapedString01() { runner.runOneTest("parseDelimitedEscapedString01") }
@Test def test_parseDelimitedEscapedString03() { runner.runOneTest("parseDelimitedEscapedString03") }