You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by sl...@apache.org on 2019/09/05 14:26:35 UTC
[incubator-daffodil] branch master updated: Add blob support
This is an automated email from the ASF dual-hosted git repository.
slawrence pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git
The following commit(s) were added to refs/heads/master by this push:
new 9de0d1a Add blob support
9de0d1a is described below
commit 9de0d1a3c3d294600d5c769243efe4f67b03bac1
Author: Steve Lawrence <sl...@apache.org>
AuthorDate: Thu Aug 15 13:02:34 2019 -0400
Add blob support
- Adds support for xs:anyURI primitive type. Elements with this type
must specifiy the new property dfdl:objectKind, which can be
"bytes" or "characters". Support for "characters" is not added in this
patch.
- On parse elements with objectKind="bytes" will have the data read
and written out to a file with a randomly generated name, with a URI
to that file being put in the infoset. On unparse, the data is read
from the URI and written to the data stream. Behavior is very similar
to xs:hexBinary.
- BLOB Objects must have lengths set with dfdl:lengthKind="explicit".
- New API functions are added to set where to write blob files to and to
a get list of all blob files that were written.
- TDML Runner is updated to be aware of xsi:type="xs:anyURI" and compare
the contents of the blob files. It will also delete blob files after a
test passes.
- The CLI will write blob files to the $PWD/daffodil-blobs/ directory.
DAFFODIL-1735
---
.github/workflows/main.yml | 2 +-
.../src/main/scala/org/apache/daffodil/Main.scala | 9 +-
.../org/apache/daffodil/dsom/ElementBase.scala | 8 +-
.../apache/daffodil/dsom/RestrictionUnion.scala | 5 +-
.../daffodil/dsom/RuntimePropertyMixins.scala | 8 +-
.../org/apache/daffodil/dsom/SimpleTypes.scala | 2 +
.../daffodil/grammar/ElementBaseGrammarMixin.scala | 27 ++
.../daffodil/grammar/primitives/Padded.scala | 3 +
.../grammar/primitives/PrimitivesLengthKind.scala | 9 +
.../org/apache/daffodil/io/DataInputStream.scala | 11 +
.../daffodil/io/InputSourceDataInputStream.scala | 12 +
.../io/StringDataInputStreamForUnparse.scala | 1 +
.../main/scala/org/apache/daffodil/util/Misc.scala | 53 +++-
.../scala/org/apache/daffodil/xml/XMLUtils.scala | 84 ++++-
.../daffodil/xml/test/unit/TestXMLUtils.scala | 78 ++++-
.../apache/daffodil/xsd/DFDL_part2_attributes.xsd | 1 +
.../org/apache/daffodil/xsd/DFDL_part3_model.xsd | 1 +
.../resources/org/apache/daffodil/xsd/dafext.xsd | 7 +
.../resources/org/apache/daffodil/xsd/dfdlx.xsd | 16 +
.../processors/unparsers/BlobLengthUnparser.scala | 147 +++++++++
.../scala/org/apache/daffodil/dpath/NodeInfo.scala | 55 +++-
.../org/apache/daffodil/infoset/InfosetImpl.scala | 1 +
.../apache/daffodil/infoset/InfosetOutputter.scala | 34 ++
.../apache/daffodil/processors/DataProcessor.scala | 9 +
.../processors/parsers/BlobLengthParsers.scala | 92 ++++++
.../daffodil/processors/parsers/PState.scala | 26 +-
.../org/apache/daffodil/tdml/TDMLRunner.scala | 66 +++-
.../tdml/processor/TDMLDFDLProcessor.scala | 8 +-
.../apache/daffodil/tdml/TDMLInfosetInputter.scala | 22 +-
.../tdml/processor/DaffodilTDMLDFDLProcessor.scala | 37 ++-
.../daffodil/section05/simple_types/Blobs.tdml | 349 +++++++++++++++++++++
.../section05/simple_types/SimpleTypes.tdml | 3 +-
.../section05/simple_types/blobs/blob_01.bin | 1 +
.../section05/simple_types/blobs/blob_02.bin | 1 +
.../section05/simple_types/blobs/blob_03.bin | 1 +
.../section05/simple_types/blobs/blob_04.bin | 1 +
.../section05/simple_types/blobs/blob_07.bin | 1 +
.../section05/simple_types/blobs/blob_13a.bin | 1 +
.../section05/simple_types/blobs/blob_13b.bin | 1 +
.../section05/simple_types/blobs/blob_13c.bin | 1 +
.../section05/simple_types/blobs/blob_13d.bin | 1 +
.../section05/simple_types/TestBlobs.scala | 56 ++++
42 files changed, 1176 insertions(+), 75 deletions(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index df527bd..1df154f 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -34,7 +34,7 @@ jobs:
############################################################
- name: Checkout Repository
- uses: actions/checkout@master
+ uses: actions/checkout@v1.0.0
- name: Install Dependencies (Linux)
run: |
diff --git a/daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala b/daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala
index 7b03b7a..ceed77d 100644
--- a/daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala
+++ b/daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala
@@ -28,6 +28,7 @@ import java.io.FileInputStream
import java.io.ByteArrayInputStream
import java.io.File
import java.nio.channels.Channels
+import java.nio.file.Paths
import java.net.URI
import java.util.Scanner
@@ -720,8 +721,12 @@ object Main extends Logging {
pf
}
+ // write blobs to $PWD/daffodil-blobs/*.bin
+ val blobDir = Paths.get(System.getProperty("user.dir"), "daffodil-blobs")
+ val blobSuffix = ".bin"
+
def getInfosetOutputter(infosetType: String, os: java.io.OutputStream): InfosetOutputter = {
- infosetType match {
+ val outputter = infosetType match {
case "xml" => new XMLTextInfosetOutputter(os, true)
case "scala-xml" => new ScalaXMLInfosetOutputter()
case "json" => new JsonInfosetOutputter(os, true)
@@ -729,6 +734,8 @@ object Main extends Logging {
case "w3cdom" => new W3CDOMInfosetOutputter()
case "null" => new NullInfosetOutputter()
}
+ outputter.setBlobAttributes(blobDir, null, blobSuffix)
+ outputter
}
// Converts the data to whatever form the InfosetInputter will want. Note
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/dsom/ElementBase.scala b/daffodil-core/src/main/scala/org/apache/daffodil/dsom/ElementBase.scala
index a4b0771..09d98a7 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/dsom/ElementBase.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/dsom/ElementBase.scala
@@ -643,6 +643,10 @@ trait ElementBase
primType match {
case PrimType.HexBinary => Representation.Binary
case PrimType.String => Representation.Text
+ case PrimType.AnyURI => objectKind match {
+ case ObjectKindType.Bytes => Representation.Binary
+ case ObjectKindType.Chars => Representation.Text
+ }
case _ => representation
}
} else {
@@ -650,7 +654,7 @@ trait ElementBase
}
rep match {
case Representation.Binary =>
- if (isComplexType || primType != PrimType.HexBinary) byteOrderEv // ensure defined
+ if (isComplexType || (primType != PrimType.HexBinary && primType != PrimType.AnyURI)) byteOrderEv // ensure defined
case _ =>
charsetEv // ensure defined
}
@@ -680,6 +684,7 @@ trait ElementBase
private def getImplicitAlignmentInBits(thePrimType: PrimType, theRepresentation: Representation): Int = {
(theRepresentation, thePrimType) match {
+ case (Representation.Text, PrimType.AnyURI) => this.subsetError("Property value objectKind='chars' is not supported.")
case (Representation.Text, PrimType.HexBinary) => Assert.impossible("type xs:hexBinary with representation='text'")
case (Representation.Text, _) => knownEncodingAlignmentInBits
case (Representation.Binary, PrimType.String) => Assert.impossible("type xs:string with representation='binary'")
@@ -687,6 +692,7 @@ trait ElementBase
case (Representation.Binary, PrimType.Float | PrimType.Boolean) => 32
case (Representation.Binary, PrimType.Double) => 64
case (Representation.Binary, PrimType.HexBinary) => 8
+ case (Representation.Binary, PrimType.AnyURI) => 8
// Handle 64 bit types
case (Representation.Binary, PrimType.Long | PrimType.UnsignedLong) =>
binaryNumberRep match {
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/dsom/RestrictionUnion.scala b/daffodil-core/src/main/scala/org/apache/daffodil/dsom/RestrictionUnion.scala
index f49475c..b09fa05 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/dsom/RestrictionUnion.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/dsom/RestrictionUnion.scala
@@ -335,11 +335,11 @@ sealed trait TypeChecks { self: Restriction =>
}
}
- // Don't need to range check String or HexBinary
+ // Don't need to range check String or HexBinary or blobs.
// no point attempting a conversion to BigDecimal so
// return early here.
primType match {
- case PrimType.String | PrimType.HexBinary => return (true, None)
+ case PrimType.String | PrimType.HexBinary | PrimType.AnyURI => return (true, None)
case _ => /* Continue on below */
}
@@ -373,6 +373,7 @@ sealed trait TypeChecks { self: Restriction =>
case PrimType.Decimal => true // Unbounded Decimal
case PrimType.HexBinary => Assert.impossibleCase // Handled earlier, shouldn't get here
case PrimType.String => Assert.impossibleCase // Handled earlier, shouldn't get here
+ case PrimType.AnyURI => Assert.impossibleCase // Handled earlier, shouldn't get here
case PrimType.NonNegativeInteger => isInNonNegativeIntegerRange(theValue)
}
val isValueWhole = {
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/dsom/RuntimePropertyMixins.scala b/daffodil-core/src/main/scala/org/apache/daffodil/dsom/RuntimePropertyMixins.scala
index 2c5b7eb..c81316a 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/dsom/RuntimePropertyMixins.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/dsom/RuntimePropertyMixins.scala
@@ -256,8 +256,8 @@ trait ElementRuntimeValuedPropertiesMixin
}
final lazy val maybeByteOrderEv = {
- if (isSimpleType && primType == PrimType.HexBinary) {
- // xs:hexBinary types should ignore the byteOrder property and
+ if (isSimpleType && (primType == PrimType.HexBinary || primType == PrimType.AnyURI)) {
+ // xs:hexBinary and xs:anyURI types should ignore the byteOrder property and
// effectively always use a bigEndian byteOrder. One way to accomplish
// this would be to modify the byteOrderExpr to return a constant value
// of "bigEndian" for hexBinary types. The problem with this is that
@@ -265,7 +265,7 @@ trait ElementRuntimeValuedPropertiesMixin
// bitOrder="leastSignificantBitFirst", which is what would happen when
// parsing hexBinary data with LSBF bitOrder, resulting in an SDE. So
// instead, we just set this to Nope, and ensure we never try to get the
- // byteOrder property when the type is xs:hexBinary. The IO layer will
+ // byteOrder property when the type is xs:hexBinary/anyURI. The IO layer will
// see this Nope and do the right thing.
Nope
} else if (optionByteOrderRaw.isDefined) {
@@ -576,7 +576,7 @@ trait ElementRuntimeValuedPropertiesMixin
}
val byteOrderExprElts =
- if (isSimpleType && primType =:= PrimType.HexBinary) {
+ if (isSimpleType && (primType =:= PrimType.HexBinary || primType =:= PrimType.AnyURI)) {
ReferencedElementInfos.None
} else {
propExprElts(optionByteOrderRaw, byteOrderEv, f)
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/dsom/SimpleTypes.scala b/daffodil-core/src/main/scala/org/apache/daffodil/dsom/SimpleTypes.scala
index ac0563a..8abbcb5 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/dsom/SimpleTypes.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/dsom/SimpleTypes.scala
@@ -170,6 +170,7 @@ object PrimitiveType {
case PrimType.DateTime => DateTime
case PrimType.Date => Date
case PrimType.Time => Time
+ case PrimType.AnyURI => AnyURI
case _ => Assert.usageError("Not a primitive type node: " + typeNode)
}
}
@@ -193,6 +194,7 @@ object PrimitiveType {
val DateTime = new PrimitiveType(PrimType.DateTime)
val Date = new PrimitiveType(PrimType.Date)
val Time = new PrimitiveType(PrimType.Time)
+ val AnyURI = new PrimitiveType(PrimType.AnyURI)
}
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/grammar/ElementBaseGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/grammar/ElementBaseGrammarMixin.scala
index 8064bba..2a4882c 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/grammar/ElementBaseGrammarMixin.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/grammar/ElementBaseGrammarMixin.scala
@@ -50,6 +50,7 @@ import org.apache.daffodil.grammar.primitives.BinaryFloat
import org.apache.daffodil.grammar.primitives.BinaryIntegerKnownLength
import org.apache.daffodil.grammar.primitives.BinaryIntegerPrefixedLength
import org.apache.daffodil.grammar.primitives.BinaryIntegerRuntimeLength
+import org.apache.daffodil.grammar.primitives.BlobSpecifiedLength
import org.apache.daffodil.grammar.primitives.CaptureContentLengthEnd
import org.apache.daffodil.grammar.primitives.CaptureContentLengthStart
import org.apache.daffodil.grammar.primitives.CaptureValueLengthEnd
@@ -692,6 +693,25 @@ trait ElementBaseGrammarMixin
}
}
+ private lazy val specifiedLengthBlob = prod("specifiedLengthBlob") {
+ lengthUnits match {
+ case LengthUnits.Bytes => BlobSpecifiedLength(this)
+ case LengthUnits.Bits => BlobSpecifiedLength(this)
+ case LengthUnits.Characters => SDE("lengthUnits='characters' is not valid for blob data.")
+ }
+ }
+
+ private lazy val blobValue = prod("blobValue") {
+ lengthKind match {
+ case LengthKind.Explicit => specifiedLengthBlob
+ case _ => SDE("objectKind='bytes' must have dfdl:lengthKind='explicit'")
+ }
+ }
+
+ private lazy val clobValue = prod("clobValue") {
+ notYetImplemented("objectKind='chars'")
+ }
+
private lazy val textInt = prod("textInt", impliedRepresentation == Representation.Text) {
standardTextInt || zonedTextInt
}
@@ -1034,6 +1054,13 @@ trait ElementBaseGrammarMixin
primType match {
case PrimType.String => stringValue
case PrimType.HexBinary => hexBinaryValue
+ case PrimType.AnyURI => {
+ val res = impliedRepresentation match {
+ case Representation.Binary => blobValue
+ case Representation.Text => clobValue
+ }
+ res
+ }
case _ => {
val res = impliedRepresentation match {
case Representation.Binary => binaryValue
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/Padded.scala b/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/Padded.scala
index 4b4e9c5..f859331 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/Padded.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/Padded.scala
@@ -119,6 +119,9 @@ trait PaddingInfoMixin {
case PrimType.HexBinary => {
(MaybeChar.Nope, TextJustificationType.None)
}
+ case PrimType.AnyURI => {
+ (MaybeChar.Nope, TextJustificationType.None)
+ }
}
theJust
}
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/PrimitivesLengthKind.scala b/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/PrimitivesLengthKind.scala
index 95fa4ef..39ac0d7 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/PrimitivesLengthKind.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/PrimitivesLengthKind.scala
@@ -30,6 +30,7 @@ import org.apache.daffodil.processors.dfa.TextDelimitedParserWithEscapeBlock
import org.apache.daffodil.processors.dfa.TextPaddingParser
import org.apache.daffodil.processors.parsers.BCDDecimalDelimitedParser
import org.apache.daffodil.processors.parsers.BCDIntegerDelimitedParser
+import org.apache.daffodil.processors.parsers.BlobSpecifiedLengthParser
import org.apache.daffodil.processors.parsers.HexBinaryDelimitedParser
import org.apache.daffodil.processors.parsers.HexBinaryEndOfBitLimitParser
import org.apache.daffodil.processors.parsers.HexBinaryLengthPrefixedParser
@@ -44,6 +45,7 @@ import org.apache.daffodil.processors.parsers.StringOfSpecifiedLengthParser
import org.apache.daffodil.processors.parsers.{ Parser => DaffodilParser }
import org.apache.daffodil.processors.unparsers.BCDDecimalDelimitedUnparser
import org.apache.daffodil.processors.unparsers.BCDIntegerDelimitedUnparser
+import org.apache.daffodil.processors.unparsers.BlobSpecifiedLengthUnparser
import org.apache.daffodil.processors.unparsers.HexBinaryLengthPrefixedUnparser
import org.apache.daffodil.processors.unparsers.HexBinaryMinLengthInBytesUnparser
import org.apache.daffodil.processors.unparsers.HexBinarySpecifiedLengthUnparser
@@ -71,6 +73,13 @@ case class HexBinarySpecifiedLength(e: ElementBase) extends Terminal(e, true) {
override lazy val unparser: DaffodilUnparser = new HexBinarySpecifiedLengthUnparser(e.elementRuntimeData, e.unparseTargetLengthInBitsEv)
}
+case class BlobSpecifiedLength(e: ElementBase) extends Terminal(e, true) {
+
+ override lazy val parser = new BlobSpecifiedLengthParser(e.elementRuntimeData, e.elementLengthInBitsEv)
+
+ override lazy val unparser = new BlobSpecifiedLengthUnparser(e.elementRuntimeData, e.unparseTargetLengthInBitsEv)
+}
+
case class StringOfSpecifiedLength(e: ElementBase) extends Terminal(e, true) with Padded {
private def erd = e.elementRuntimeData
diff --git a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataInputStream.scala b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataInputStream.scala
index 5af3556..c2178ee 100644
--- a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataInputStream.scala
+++ b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataInputStream.scala
@@ -287,6 +287,17 @@ trait DataInputStream
def getByteArray(bitLengthFrom1: Int, finfo: FormatInfo): Array[Byte]
/**
+ * Same as getByteArray(Int, FormatInfo), except it accepts the array as a
+ * parameter and will fill that array with bytes. This can help to improve
+ * performance if an array can be reused to avoid allocations. But the caller
+ * must be careful upon return, since the array will not be cleared and may
+ * be bigger than the number of bits filled. It is the responsibility of the
+ * caller to know which bits in the array were set based on the bit length
+ * and format info.
+ */
+ def getByteArray(bitLengthFrom1: Int, finfo: FormatInfo, array: Array[Byte]): Unit
+
+ /**
* Returns a long integer containing the bits between the current bit position
* and that position plus the bitLength.
* <p>
diff --git a/daffodil-io/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala b/daffodil-io/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala
index 124bb2c..064a821 100644
--- a/daffodil-io/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala
+++ b/daffodil-io/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala
@@ -154,6 +154,18 @@ final class InputSourceDataInputStream private (val inputSource: InputSource)
array
}
+ def getByteArray(bitLengthFrom1: Int, finfo: FormatInfo, array: Array[Byte]): Unit = {
+ // threadCheck()
+ if (!isDefinedForLength(bitLengthFrom1)) throw DataInputStream.NotEnoughDataException(bitLengthFrom1)
+
+ val bytesNeeded = (bitLengthFrom1 + 7) / 8
+ Assert.usage(array.size >= bytesNeeded)
+
+ fillByteArray(array, bitLengthFrom1, finfo)
+
+ setBitPos0b(bitPos0b + bitLengthFrom1)
+ }
+
/**
* Accepts a preallocated array and a bitLength. Reads the specified number
* of bits and stores them in the array consistent with bitOrder and
diff --git a/daffodil-io/src/main/scala/org/apache/daffodil/io/StringDataInputStreamForUnparse.scala b/daffodil-io/src/main/scala/org/apache/daffodil/io/StringDataInputStreamForUnparse.scala
index 9033e58..eb96523 100644
--- a/daffodil-io/src/main/scala/org/apache/daffodil/io/StringDataInputStreamForUnparse.scala
+++ b/daffodil-io/src/main/scala/org/apache/daffodil/io/StringDataInputStreamForUnparse.scala
@@ -72,6 +72,7 @@ final class StringDataInputStreamForUnparse
override def getUnsignedBigInt(bitLengthFrom1: Int, finfo: FormatInfo): BigInt = doNotUse
override def getUnsignedLong(bitLengthFrom1To64: Int, finfo: FormatInfo): passera.unsigned.ULong = doNotUse
override def getByteArray(bitLengthFrom1: Int, finfo: FormatInfo): Array[Byte] = doNotUse
+ override def getByteArray(bitLengthFrom1: Int, finfo: FormatInfo, array: Array[Byte]): Unit = doNotUse
override def pastData(nBytesRequested: Int): java.nio.ByteBuffer = doNotUse
override def setBitLimit0b(bitLimit0b: MaybeULong): Boolean = doNotUse
override def setDebugging(setting: Boolean): Unit = doNotUse
diff --git a/daffodil-lib/src/main/scala/org/apache/daffodil/util/Misc.scala b/daffodil-lib/src/main/scala/org/apache/daffodil/util/Misc.scala
index 4632904..1eedccf 100644
--- a/daffodil-lib/src/main/scala/org/apache/daffodil/util/Misc.scala
+++ b/daffodil-lib/src/main/scala/org/apache/daffodil/util/Misc.scala
@@ -17,20 +17,24 @@
package org.apache.daffodil.util
-import java.io.InputStream
-import java.io.File
-import java.net.URL
-import java.net.URI
import java.io.ByteArrayInputStream
-import java.nio.channels.ReadableByteChannel
-import java.nio.channels.WritableByteChannel
import java.io.ByteArrayOutputStream
+import java.io.File
+import java.io.InputStream
+import java.net.URI
+import java.net.URL
import java.net.URLClassLoader
import java.nio.ByteBuffer
import java.nio.CharBuffer
-import java.nio.charset.{ Charset => JavaCharset }
+import java.nio.channels.ReadableByteChannel
+import java.nio.channels.WritableByteChannel
import java.nio.charset.CodingErrorAction
+import java.nio.charset.{ Charset => JavaCharset }
+import java.nio.file.Files
+import java.nio.file.Paths
+
import scala.collection.JavaConverters._
+
import org.apache.daffodil.equality._
/**
@@ -121,6 +125,41 @@ object Misc {
}
}
+ /**
+ * Search for a resource name, trying a handful of heuristics.
+ *
+ * This is useful in cases where it's okay to be a little lax when searching
+ * for a resource name (e.g. testing). This is often needed in situations
+ * like directory structures not exactly match paths and we just want to try
+ * our best to find a file. Heuristics include things like ignoring leading
+ * directories in a resource, looking for a resource as a file, etc. This
+ * does mean that it's possible that this could find the wrong file if there
+ * are ambiguities, so be careful when using.
+ *
+ * If ambiguities would be a serious problem, use getResourceOption or
+ * getResourceRelativeOption.
+ */
+ def searchResourceOption(resName: String, relativeTo: Option[URI]): Option[URI] = {
+ val resAsURI = new URI(resName)
+ val resPath =
+ if (resAsURI.getScheme != null) Paths.get(resAsURI) else Paths.get(resName)
+ val resolvedURI =
+ if (Files.exists(resPath)) Some(resPath.toFile().toURI())
+ else Misc.getResourceRelativeOption(resName, relativeTo)
+ val res = resolvedURI.orElse {
+ // try ignoring the directory part
+ val parts = resName.split("/")
+ if (parts.length > 1) { // if there is one
+ val filePart = parts.last
+ val secondTry = searchResourceOption(filePart, relativeTo) // recursively
+ secondTry
+ } else {
+ None
+ }
+ }
+ res
+ }
+
private def tryURL(url: URL): Option[URI] = {
var is: InputStream = null
val res =
diff --git a/daffodil-lib/src/main/scala/org/apache/daffodil/xml/XMLUtils.scala b/daffodil-lib/src/main/scala/org/apache/daffodil/xml/XMLUtils.scala
index 4fd7512..880ee66 100644
--- a/daffodil-lib/src/main/scala/org/apache/daffodil/xml/XMLUtils.scala
+++ b/daffodil-lib/src/main/scala/org/apache/daffodil/xml/XMLUtils.scala
@@ -18,6 +18,10 @@
package org.apache.daffodil.xml
import java.io.File
+import java.net.URI
+import java.nio.file.Files
+import java.nio.file.Paths
+import java.nio.file.StandardOpenOption
import scala.collection.mutable
import scala.collection.mutable.ArrayBuilder
@@ -712,15 +716,20 @@ object XMLUtils {
Assert.invariant(pre != "")
false // drop dafint attributes.
}
- case xsiTypeAttr @ PrefixedAttribute(_, "type", _, _) if (NS(xsiTypeAttr.getNamespace(e)) == XMLUtils.XSI_NAMESPACE) =>
- false // drop xsi:type attributes for now. Such time as we add
- // support for them, we would need to not remove them.
- // TODO: actually check xsi:type attributes are correct - but this
- // requires schema-aware comparison.
- case xsiTypeAttr @ PrefixedAttribute("xsi", "type", _, _) =>
- false // drop xsi:type attributes for now. Even if prefix xsi is not defined.
- // This just avoids having to edit many tests to add in the xmlns:xsi=....
- // namespace declaration.
+ case xsiTypeAttr @ PrefixedAttribute(_, "type", _, _) if (NS(xsiTypeAttr.getNamespace(e)) == XMLUtils.XSI_NAMESPACE) => {
+ // TODO: actually check xsi:type attributes are correct according
+ // to the schema--requires schema-awareness in TDML Runner.
+ // Do not hide xsi:type since it is used for hints for type aware
+ // comparisons.
+ true
+ }
+ case xsiTypeAttr @ PrefixedAttribute("xsi", "type", _, _) => {
+ // TODO: actually check xsi:type attributes are correct according
+ // to the schema--requires schema-awareness in TDML Runner.
+ // Do not hide xsi:type since it is used for hints for type aware
+ // comparisons.
+ true
+ }
case attr =>
true // keep all other attributes
}
@@ -898,13 +907,68 @@ Differences were (path, expected, actual):
computeTextDiff(zPath, dataA, dataB, maybeType)
}
+ def computeBlobDiff(zPath: String, dataA: String, dataB: String) = {
+ val uriA = Misc.searchResourceOption(dataA, None)
+ val uriB = Misc.searchResourceOption(dataB, None)
+
+ val pathA = uriA.map { u => Paths.get(u) }
+ val pathB = uriB.map { u => Paths.get(u) }
+
+ val canReadA = pathA.map { p => Files.isReadable(p) }.getOrElse(false)
+ val canReadB = pathB.map { p => Files.isReadable(p) }.getOrElse(false)
+
+ if (!canReadA || !canReadB) {
+ val path = zPath + ".canRead"
+ Seq((path, canReadA.toString, canReadB.toString))
+ } else {
+ val CHUNK_SIZE = 1024
+ val arrayA = new Array[Byte](CHUNK_SIZE)
+ val arrayB = new Array[Byte](CHUNK_SIZE)
+
+ val streamA = Files.newInputStream(pathA.get, StandardOpenOption.READ)
+ val streamB = Files.newInputStream(pathB.get, StandardOpenOption.READ)
+
+ var lenA: Int = 0
+ var lenB: Int = 0
+ var numSameBytes: Int = 0
+ var areSame: Boolean = true
+
+ while ({
+ lenA = streamA.read(arrayA)
+ lenB = streamB.read(arrayB)
+ areSame = lenA == lenB && arrayA.sameElements(arrayB)
+ areSame && lenA != -1 && lenB != -1
+ }) {
+ numSameBytes += lenA
+ }
+
+ if (!areSame) {
+ val zip = arrayA.zip(arrayB)
+ val firstDiffIndex = zip.indexWhere(z => z._1 != z._2)
+
+ val MAX_CONTEXT = 40
+ val contextA = arrayA.take(lenA).drop(firstDiffIndex).take(MAX_CONTEXT)
+ val contextB = arrayB.take(lenB).drop(firstDiffIndex).take(MAX_CONTEXT)
+ val hexA = Misc.bytes2Hex(contextA)
+ val hexB = Misc.bytes2Hex(contextB)
+
+ val absoluteIndex = numSameBytes + firstDiffIndex
+ val path = zPath + ".bytesAt(" + (absoluteIndex + 1) + ")"
+ Seq((path, hexA, hexB))
+ } else {
+ Nil
+ }
+ }
+ }
+
def computeTextDiff(
zPath: String,
dataA: String,
dataB: String,
maybeType: Option[String]): Seq[(String, String, String)] = {
- if (textIsSame(dataA, dataB, maybeType)) Nil
+ if (maybeType.isDefined && maybeType.get == "xs:anyURI") computeBlobDiff(zPath, dataA, dataB)
+ else if (textIsSame(dataA, dataB, maybeType)) Nil
else {
// There must be some difference, so let's find just the first index of
// difference and we'll include that and some following characters for
diff --git a/daffodil-lib/src/test/scala/org/apache/daffodil/xml/test/unit/TestXMLUtils.scala b/daffodil-lib/src/test/scala/org/apache/daffodil/xml/test/unit/TestXMLUtils.scala
index 4bf17dc..ed2486a 100644
--- a/daffodil-lib/src/test/scala/org/apache/daffodil/xml/test/unit/TestXMLUtils.scala
+++ b/daffodil-lib/src/test/scala/org/apache/daffodil/xml/test/unit/TestXMLUtils.scala
@@ -17,13 +17,21 @@
package org.apache.daffodil.xml.test.unit
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.StandardOpenOption
+
import scala.xml._
-import org.apache.daffodil.xml.XMLUtils
-import org.apache.daffodil.xml.JDOMUtils
-import org.apache.daffodil.xml.NS
+
import junit.framework.Assert._
import org.junit.Test
+
import org.apache.daffodil.Implicits._
+import org.apache.daffodil.util.Misc
+import org.apache.daffodil.xml.JDOMUtils
+import org.apache.daffodil.xml.NS
+import org.apache.daffodil.xml.XMLUtils
class TestXMLUtils {
@@ -261,4 +269,68 @@ class TestXMLUtils {
val expected = "€‚ƒ„…†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ ¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
assertEquals(expected, actual)
}
+
+ private def createBlobFile(blob: String): Path = {
+ val dir = Paths.get(System.getProperty("java.io.tmpdir"), "daffodil-test", "blobs")
+ Files.createDirectories(dir)
+ val path = Files.createTempFile(dir, null, null)
+ val bytes = Misc.hex2Bytes(blob)
+ val stream = Files.newOutputStream(path, StandardOpenOption.WRITE)
+ stream.write(bytes)
+ stream.close()
+ path
+ }
+
+ @Test def testBlobDiff01() {
+ val path1 = createBlobFile("A1B2C3")
+ val path2 = createBlobFile("A1B2C3")
+ val diff = XMLUtils.computeBlobDiff("path", path1.toString, path2.toString)
+ assertEquals(diff, Nil)
+ Files.delete(path1)
+ Files.delete(path2)
+ }
+
+ @Test def testBlobDiff02() {
+ val path1 = createBlobFile("A1B2C3D4")
+ val path2 = createBlobFile("A1B1C3")
+ val diff = XMLUtils.computeBlobDiff("path", path1.toString, path2.toString)
+ assertEquals("path.bytesAt(2)", diff(0)._1)
+ assertEquals("B2C3D4", diff(0)._2)
+ assertEquals("B1C3", diff(0)._3)
+ Files.delete(path1)
+ Files.delete(path2)
+ }
+
+ @Test def testBlobDiff03() {
+ val path1 = createBlobFile("A1B2C3D4")
+ val path2 = createBlobFile("")
+ val diff = XMLUtils.computeBlobDiff("path", path1.toString, path2.toString)
+ assertEquals("path.bytesAt(1)", diff(0)._1)
+ assertEquals("A1B2C3D4", diff(0)._2)
+ assertEquals("", diff(0)._3)
+ Files.delete(path1)
+ Files.delete(path2)
+ }
+
+ @Test def testBlobDiff04() {
+ val path1 = createBlobFile("A1B2C3D4")
+ val path2 = Paths.get("file://does/not/exist")
+ val diff = XMLUtils.computeBlobDiff("path", path1.toString, path2.toString)
+ assertEquals("path.canRead", diff(0)._1)
+ assertEquals("true", diff(0)._2)
+ assertEquals("false", diff(0)._3)
+ Files.delete(path1)
+ }
+
+ @Test def testBlobDiff05() {
+ val path1 = createBlobFile(("00" * 1024) + ("A1" * 41))
+ val path2 = createBlobFile(("00" * 1024))
+ val diff = XMLUtils.computeBlobDiff("path", path1.toString, path2.toString)
+ println(diff)
+ assertEquals("path.bytesAt(1025)", diff(0)._1)
+ assertEquals("A1" * 40, diff(0)._2)
+ assertEquals("", diff(0)._3)
+ Files.delete(path1)
+ Files.delete(path2)
+ }
}
diff --git a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part2_attributes.xsd b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part2_attributes.xsd
index cebc507..4b9780d 100644
--- a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part2_attributes.xsd
+++ b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part2_attributes.xsd
@@ -1066,6 +1066,7 @@
<xsd:attributeGroup ref="dfdl:BooleanTextAGQualified" />
<xsd:attributeGroup ref="dfdl:BooleanBinaryAGQualified" />
<xsd:attributeGroup ref="dfdlx:SimpleTypeValueCalcAGQualified" />
+ <xsd:attributeGroup ref="dfdlx:ObjectAGQualified" />
</xsd:attributeGroup>
<xsd:attributeGroup name="LayeringAGQualified">
diff --git a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part3_model.xsd b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part3_model.xsd
index 524087e..101b26c 100644
--- a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part3_model.xsd
+++ b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part3_model.xsd
@@ -365,6 +365,7 @@
<xsd:attributeGroup ref="dfdl:BooleanTextAG" />
<xsd:attributeGroup ref="dfdl:BooleanBinaryAG" />
<xsd:attributeGroup ref="dfdlx:SimpleTypeValueCalcAG" />
+ <xsd:attributeGroup ref="dfdlx:ObjectAG" />
</xsd:attributeGroup>
</xsd:schema>
diff --git a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd
index 9b7884d..6923c7c 100644
--- a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd
+++ b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd
@@ -112,6 +112,13 @@
</xs:documentation>
</xs:annotation>
</xs:element>
+ <xs:element name="blobChunkSizeInBytes" type="xs:int" default="4096" minOccurs="0">
+ <xs:annotation>
+ <xs:documentation>
+ When reading/writing blob data, the maximum number of bytes to read/write at a time.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:element>
<xs:element name="defaultInitialRegexMatchLimitInChars" type="xs:int" default="32" minOccurs="0">
<xs:annotation>
<xs:documentation>
diff --git a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dfdlx.xsd b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dfdlx.xsd
index 7bee32b..2baac58 100644
--- a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dfdlx.xsd
+++ b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dfdlx.xsd
@@ -39,6 +39,7 @@
<xs:enumeration value="dfdlx:choiceDispatchKeyKind" />
<xs:enumeration value="dfdlx:emptyElementParsePolicy"/>
<xs:enumeration value="dfdlx:inputTypeCalc"/>
+ <xs:enumeration value="dfdlx:objectKind"/>
<xs:enumeration value="dfdlx:outputTypeCalc"/>
<xs:enumeration value="dfdlx:parseUnparsePolicy"/>
<xs:enumeration value="dfdlx:repType"/>
@@ -119,4 +120,19 @@
<xs:attribute form="qualified" name="layerBoundaryMark" type="dfdl:ListOfDFDLStringLiteral_Or_DFDLExpression" />
</xs:attributeGroup>
+ <xs:attributeGroup name="ObjectAG">
+ <xs:attribute form="qualified" name="ObjectKind" type="dfdlx:ObjectKindType" />
+ </xs:attributeGroup>
+
+ <xs:attributeGroup name="ObjectAGQualified">
+ <xs:attribute form="qualified" name="objectKind" type="dfdlx:ObjectKindType" />
+ </xs:attributeGroup>
+
+ <xs:simpleType name="ObjectKindType">
+ <xs:restriction base="xs:string">
+ <xs:enumeration value="bytes"/>
+ <xs:enumeration value="chars"/>
+ </xs:restriction>
+ </xs:simpleType>
+
</xs:schema>
diff --git a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/BlobLengthUnparser.scala b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/BlobLengthUnparser.scala
new file mode 100644
index 0000000..b991e6f
--- /dev/null
+++ b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/BlobLengthUnparser.scala
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.processors.unparsers
+
+import passera.unsigned.ULong
+
+import java.net.URI
+import java.nio.file.Files
+import java.nio.file.Paths
+import java.nio.file.StandardOpenOption
+
+import org.apache.daffodil.infoset.RetryableException
+import org.apache.daffodil.processors.ElementRuntimeData
+import org.apache.daffodil.processors.UnparseTargetLengthInBitsEv
+import org.apache.daffodil.util.Maybe._
+
+abstract class BlobUnparserBase(override val context: ElementRuntimeData)
+ extends PrimUnparser {
+
+ override lazy val runtimeDependencies = Vector()
+
+ protected def getLengthInBits(state: UState): Long
+
+ override def unparse(state: UState): Unit = {
+
+ val node = state.currentInfosetNode.asSimple
+ val value = node.dataValue.asInstanceOf[URI]
+ val lengthInBits = getLengthInBits(state)
+
+ val lengthInBytes = (lengthInBits + 7) / 8
+
+ val uriScheme = value.getScheme
+ val uriPath = value.getPath
+
+ if (uriScheme != "file") {
+ UnparseError(
+ One(context.schemaFileLocation),
+ One(state.currentLocation),
+ "Blob URI must be a file: %s", value.toString)
+ }
+
+ val path = Paths.get(value)
+
+ val fileSizeInBytes = try {
+ Files.size(path)
+ } catch {
+ case e: Exception =>
+ UnparseError(
+ One(context.schemaFileLocation),
+ One(state.currentLocation),
+ "Unable to open blob for reading: %s",
+ value.toString)
+ }
+
+ if (fileSizeInBytes > lengthInBytes) {
+ UnparseError(
+ One(context.schemaFileLocation),
+ One(state.currentLocation),
+ "Blob length (%d bits) exceeds explicit length value: %d bits",
+ fileSizeInBytes * 8,
+ lengthInBits)
+ }
+
+ val blobStream =
+ try {
+ Files.newInputStream(path, StandardOpenOption.READ)
+ } catch {
+ case e: Exception =>
+ UnparseError(
+ One(context.schemaFileLocation),
+ One(state.currentLocation),
+ "Unable to open blob for reading: %s",
+ value.toString)
+ }
+
+ val dos = state.dataOutputStream
+
+ val array = new Array[Byte](state.tunable.blobChunkSizeInBytes)
+
+ var remainingBitsToPut = lengthInBits
+ var fileHasData = true
+ while (remainingBitsToPut > 0 && fileHasData) {
+ val bitsToRead = Math.min(remainingBitsToPut, state.tunable.blobChunkSizeInBytes * 8)
+ val bytesToRead = (bitsToRead + 7) / 8
+ val bytesRead = blobStream.read(array, 0, bytesToRead.toInt)
+ if (bytesRead == -1) {
+ fileHasData = false
+ } else {
+ val bitsToPut = Math.min(bytesRead * 8, bitsToRead)
+ val ret = dos.putByteArray(array, bitsToPut.toInt, state)
+ if (!ret) {
+ blobStream.close()
+ UnparseError(
+ One(context.schemaFileLocation),
+ One(state.currentLocation),
+ "Failed to write %d blob bits",
+ lengthInBits)
+ }
+ remainingBitsToPut -= bitsToPut
+ }
+ }
+
+ blobStream.close()
+
+ // calculate the skip bits
+ val nFillBits = remainingBitsToPut
+ if (nFillBits > 0) {
+ val ret = dos.skip(nFillBits, state)
+ if (!ret) {
+ UnparseError(Nope, One(state.currentLocation), "Failed to skip %d bits.", nFillBits)
+ }
+ }
+ }
+}
+
+final class BlobSpecifiedLengthUnparser(erd: ElementRuntimeData, val lengthEv: UnparseTargetLengthInBitsEv)
+ extends BlobUnparserBase(erd) {
+
+ override def getLengthInBits(state: UState): Long = {
+ val l: Long = try {
+ lengthEv.evaluate(state).getULong.toLong
+ } catch {
+ case e: RetryableException => {
+ val uri = state.currentInfosetNode.asSimple.dataValue.asInstanceOf[URI]
+ val path = Paths.get(uri)
+ val len = Files.size(path) * 8
+ len
+ }
+ }
+ l
+ }
+}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/dpath/NodeInfo.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/dpath/NodeInfo.scala
index 0dc69a9..6246935 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/dpath/NodeInfo.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/dpath/NodeInfo.scala
@@ -17,28 +17,29 @@
package org.apache.daffodil.dpath
+import java.lang.{ Boolean => JBoolean }
+import java.lang.{ Byte => JByte }
+import java.lang.{ Double => JDouble }
+import java.lang.{ Float => JFloat }
+import java.lang.{ Integer => JInt }
+import java.lang.{ Long => JLong }
+import java.lang.{ Short => JShort }
+import java.math.{ BigDecimal => JBigDecimal }
+import java.math.{ BigInteger => JBigInt }
+import java.net.URI
+
import scala.BigDecimal
import scala.BigInt
+
import org.apache.daffodil.calendar._
import org.apache.daffodil.exceptions.Assert
import org.apache.daffodil.util.Enum
import org.apache.daffodil.util.Misc
import org.apache.daffodil.xml.GlobalQName
+import org.apache.daffodil.xml.NoNamespace
import org.apache.daffodil.xml.QName
import org.apache.daffodil.xml.RefQName
import org.apache.daffodil.xml.XMLUtils
-import java.lang.{
- Long => JLong,
- Double => JDouble,
- Float => JFloat,
- Integer => JInt,
- Short => JShort,
- Byte => JByte,
- Boolean => JBoolean
-}
-import org.apache.daffodil.exceptions.Assert
-import java.math.{ BigDecimal => JBigDecimal, BigInteger => JBigInt }
-import org.apache.daffodil.xml.NoNamespace
/**
* We need to have a data structure that lets us represent a type, and
@@ -202,6 +203,7 @@ object NodeInfo extends Enum {
case x: Double => NodeInfo.Double
case x: Float => NodeInfo.Float
case x: Array[Byte] => NodeInfo.HexBinary
+ case x: URI => NodeInfo.AnyURI
case x: Boolean => NodeInfo.Boolean
case x: DFDLCalendar => NodeInfo.DateTime
case _ => Assert.usageError("Unsupported object representation type: %s".format(a))
@@ -241,9 +243,22 @@ object NodeInfo extends Enum {
Date, Time, DateTime,
UnsignedByte, Byte,
HexBinary,
+ AnyURI,
String, NonEmptyString), Nil)
- with Boolean.Kind with Complex.Kind with Array.Kind
- with ArrayIndex.Kind with Double.Kind with Float.Kind with Date.Kind with Time.Kind with DateTime.Kind with UnsignedByte.Kind with Byte.Kind with HexBinary.Kind with NonEmptyString.Kind
+ with Boolean.Kind
+ with Complex.Kind
+ with Array.Kind
+ with ArrayIndex.Kind
+ with Double.Kind
+ with Float.Kind
+ with Date.Kind
+ with Time.Kind
+ with DateTime.Kind
+ with UnsignedByte.Kind
+ with Byte.Kind
+ with HexBinary.Kind
+ with NonEmptyString.Kind
+ with AnyURI.Kind
/**
* All complex types are represented by this one type object.
@@ -277,7 +292,7 @@ object NodeInfo extends Enum {
}
protected sealed trait AnyAtomicKind extends AnySimpleType.Kind
- case object AnyAtomic extends TypeNode(AnySimpleType, Seq(String, Numeric, Boolean, Opaque, AnyDateTime)) with AnyAtomicKind {
+ case object AnyAtomic extends TypeNode(AnySimpleType, Seq(String, Numeric, Boolean, Opaque, AnyDateTime, AnyURI)) with AnyAtomicKind {
type Kind = AnyAtomicKind
}
@@ -338,6 +353,7 @@ object NodeInfo extends Enum {
val Double = PrimType.Double
val Float = PrimType.Float
val HexBinary = PrimType.HexBinary
+ val AnyURI = PrimType.AnyURI
val Boolean = PrimType.Boolean
val DateTime = PrimType.DateTime
val Date = PrimType.Date
@@ -359,6 +375,7 @@ object NodeInfo extends Enum {
Double,
Float,
HexBinary,
+ AnyURI,
Boolean,
DateTime,
Date,
@@ -494,6 +511,12 @@ object NodeInfo extends Enum {
override def fromXMLString(s: String): JBoolean = s.toBoolean
}
+ protected sealed trait AnyURIKind extends AnySimpleType.Kind
+ case object AnyURI extends PrimTypeNode(AnyAtomic) with AnyURIKind {
+ type Kind = AnyURIKind
+ override def fromXMLString(s: String): URI = new URI(s)
+ }
+
protected sealed trait HexBinaryKind extends Opaque.Kind
case object HexBinary extends PrimTypeNode(Opaque) with HexBinaryKind {
type Kind = HexBinaryKind
@@ -538,7 +561,7 @@ object NodeInfo extends Enum {
private lazy val allDFDLTypes = List(
Float, Double, Decimal, Integer, Long, Int, Short, Byte,
NonNegativeInteger, UnsignedLong, UnsignedInt, UnsignedShort, UnsignedByte,
- String, Boolean, HexBinary,
+ String, Boolean, HexBinary, AnyURI,
Date, Time, DateTime)
lazy val allTypes =
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetImpl.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetImpl.scala
index ae0a971..4b2b6e1 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetImpl.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetImpl.scala
@@ -1146,6 +1146,7 @@ sealed class DISimple(override val erd: ElementRuntimeData)
case b: JBoolean => b
case _: AtomicLong | _: AtomicInteger => Assert.invariantFailed("Unsupported type. %s of type %s.".format(x, Misc.getNameFromClass(x)))
case n: java.lang.Number => n
+ case u: java.net.URI => u
case ar: AnyRef => Assert.invariantFailed("Unsupported type. %s of type %s.".format(x, Misc.getNameFromClass(x)))
}
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetOutputter.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetOutputter.scala
index 5c0c195..8f12913 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetOutputter.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/infoset/InfosetOutputter.scala
@@ -17,6 +17,8 @@
package org.apache.daffodil.infoset
+import java.nio.file.Path
+import java.nio.file.Paths
trait InfosetOutputter {
@@ -132,6 +134,38 @@ trait InfosetOutputter {
val maybeIsNilled = diElement.maybeIsNilled
maybeIsNilled.isDefined && maybeIsNilled.get == true
}
+
+ /**
+ * Set the attributes for how to create blob files.
+ *
+ * @param dir the Path the the directory to create files. If the directory
+ * does not exist, Daffodil will attempt to create it before
+ * writing a blob.
+ * @param prefix the prefix string to be used in generating a blob file name
+ * @param suffix the suffix string to be used in generating a blob file name
+ */
+ final def setBlobAttributes(dir: Path, prefix: String, suffix: String): Unit = {
+ blobDirectory = dir
+ blobPrefix = prefix
+ blobSuffix = suffix
+ }
+
+ /**
+ * Get the list of blob paths that were output in the infoset.
+ *
+ * This is the same as what would be found by iterating over the infoset.
+ */
+ final def getBlobPaths(): Seq[Path] = blobPaths
+
+
+ final def getBlobDirectory(): Path = blobDirectory
+ final def getBlobPrefix(): String = blobPrefix
+ final def getBlobSuffix(): String = blobSuffix
+ final def setBlobPaths(paths: Seq[Path]): Unit = blobPaths = paths
+ private var blobDirectory: Path = Paths.get(System.getProperty("java.io.tmpdir"))
+ private var blobPrefix: String = "daffodil-"
+ private var blobSuffix: String = ".blob"
+ private var blobPaths: Seq[Path] = Seq.empty
}
object Status extends Enumeration {
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DataProcessor.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DataProcessor.scala
index e318c95..494f55e 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DataProcessor.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DataProcessor.scala
@@ -21,6 +21,7 @@ import java.io.File
import java.io.ObjectOutputStream
import java.nio.channels.Channels
import java.nio.CharBuffer
+import java.nio.file.Files
import java.nio.LongBuffer
import java.util.zip.GZIPOutputStream
@@ -205,6 +206,14 @@ class DataProcessor(val ssrd: SchemaSetRuntimeData)
// so that we can start to throw away infoset nodes. When that happens
// we can remove this line.
state.infoset.visit(state.output)
+ state.output.setBlobPaths(state.blobPaths)
+ } else {
+ // failed, so delete all blobs that were created
+ state.blobPaths.foreach { path =>
+ Files.delete(path)
+ }
+ // ensure the blob paths are empty in case of outputter reuse
+ state.output.setBlobPaths(Seq.empty)
}
val s = state
val dp = s.dataProc
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/BlobLengthParsers.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/BlobLengthParsers.scala
new file mode 100644
index 0000000..6de45a7
--- /dev/null
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/BlobLengthParsers.scala
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.processors.parsers
+
+import java.nio.file.Files
+import java.nio.file.StandardOpenOption
+
+import org.apache.daffodil.processors.ElementRuntimeData
+import org.apache.daffodil.processors.LengthInBitsEv
+import org.apache.daffodil.util.MaybeULong
+
+sealed abstract class BlobLengthParser(override val context: ElementRuntimeData)
+ extends PrimParser {
+
+ protected def getLengthInBits(pstate: PState): Long
+
+ override final def parse(start: PState): Unit = {
+ val dis = start.dataInputStream
+ val currentElement = start.simpleElement
+ val nBits = getLengthInBits(start)
+
+ val blobPath = try {
+ val blobDir = start.output.getBlobDirectory
+ Files.createDirectories(blobDir)
+ Files.createTempFile(blobDir, start.output.getBlobPrefix, start.output.getBlobSuffix)
+ } catch {
+ case e: Exception => start.SDE("Unable to create blob file: ", e.getMessage)
+ }
+ val blobStream = Files.newOutputStream(blobPath, StandardOpenOption.WRITE)
+
+ var remainingBitsToGet = nBits
+
+ val array = new Array[Byte](start.tunable.blobChunkSizeInBytes)
+ val blobChunkSizeInBits = start.tunable.blobChunkSizeInBytes * 8
+
+ while (remainingBitsToGet > 0) {
+ val bitsToGet = Math.min(remainingBitsToGet, blobChunkSizeInBits).toInt
+ if (dis.isDefinedForLength(bitsToGet)) {
+ start.dataInputStream.getByteArray(bitsToGet, start, array)
+ val bytesToPut = (bitsToGet + 7) / 8
+ blobStream.write(array, 0, bytesToPut)
+ remainingBitsToGet -= bitsToGet
+ } else {
+ val remainingBits =
+ if (dis.remainingBits.isDefined) {
+ val totalBitsRead = nBits - remainingBitsToGet
+ MaybeULong(dis.remainingBits.get + totalBitsRead)
+ } else {
+ MaybeULong.Nope
+ }
+ PENotEnoughBits(start, nBits, remainingBits)
+ remainingBitsToGet = 0 // break out of the loop
+ }
+ }
+
+ blobStream.close()
+
+ if (start.isSuccess) {
+ currentElement.setDataValue(blobPath.toUri)
+ start.addBlobPath(blobPath)
+ } else {
+ Files.delete(blobPath)
+ }
+
+ }
+}
+
+final class BlobSpecifiedLengthParser(erd: ElementRuntimeData, lengthEv: LengthInBitsEv)
+ extends BlobLengthParser(erd) {
+
+ override val runtimeDependencies = Vector(lengthEv)
+
+ override def getLengthInBits(pstate: PState): Long = {
+ lengthEv.evaluate(pstate).get
+ }
+
+}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/PState.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/PState.scala
index 23b1ef2..dda6caa 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/PState.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/PState.scala
@@ -18,6 +18,8 @@
package org.apache.daffodil.processors.parsers
import java.nio.channels.Channels
+import java.nio.file.Files
+import java.nio.file.Path
import scala.Right
import scala.collection.mutable
@@ -153,6 +155,7 @@ final class PState private (
val mpstate: MPState,
dataProcArg: DataProcessor,
var delimitedParseResult: Maybe[dfa.ParseResult],
+ var blobPaths: Seq[Path],
tunable: DaffodilTunables) // Runtime tunables obtained from DataProcessor)
extends ParseOrUnparseState(vmap, diagnosticsArg, One(dataProcArg), tunable) {
@@ -276,6 +279,10 @@ final class PState private (
discriminatorStack.push(disc)
}
+ def addBlobPath(path: Path) {
+ blobPaths = path +: blobPaths
+ }
+
final def notifyDebugging(flag: Boolean) {
// threadCheck()
dataInputStream.setDebugging(flag)
@@ -392,6 +399,7 @@ object PState {
var validationStatus: Boolean = _
var diagnostics: List[Diagnostic] = _
var delimitedParseResult: Maybe[dfa.ParseResult] = Nope
+ var blobPaths: Seq[Path] = Seq.empty
val mpStateMark = new MPState.Mark
@@ -405,6 +413,7 @@ object PState {
diagnostics = null
delimitedParseResult = Nope
mpStateMark.clear()
+ blobPaths = Seq.empty
// DO NOT clear requestorId. It is there to help us debug if we try to repeatedly reset/discard a mark already discarded.
}
@@ -420,6 +429,7 @@ object PState {
this.validationStatus = ps.validationStatus
this.diagnostics = ps.diagnostics
this.mpStateMark.captureFrom(ps.mpstate)
+ this.blobPaths = ps.blobPaths
}
def restoreInfoset(ps: PState) = {
@@ -439,6 +449,20 @@ object PState {
ps.diagnostics = this.diagnostics
ps.delimitedParseResult = this.delimitedParseResult
mpStateMark.restoreInto(ps.mpstate)
+
+ // We are backtracking here, potentially past blob files that have
+ // already been written, so we must delete them. Since we always prepend
+ // blobs to the blobPaths Seq as they are created, we can delete them by
+ // deleting all blob files that are in front of the blobsToKeep Seq. This
+ // also lets us do fast reference equality comparisons for determining
+ // when to stop deleting.
+ val blobsToKeep = this.blobPaths
+ var currentBlobs = ps.blobPaths
+ while (currentBlobs ne blobsToKeep) {
+ Files.delete(currentBlobs.head)
+ currentBlobs = currentBlobs.tail
+ }
+ ps.blobPaths = blobsToKeep
}
}
@@ -478,7 +502,7 @@ object PState {
dis.cst.setPriorBitOrder(root.defaultBitOrder)
val newState = new PState(doc.asInstanceOf[DIElement], dis, output, variables, diagnostics, mutablePState,
- dataProc.asInstanceOf[DataProcessor], Nope, tunables)
+ dataProc.asInstanceOf[DataProcessor], Nope, Seq.empty, tunables)
newState
}
diff --git a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala
index e1bccc1..43c6748 100644
--- a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala
+++ b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/TDMLRunner.scala
@@ -409,22 +409,7 @@ class DFDLTestSuite private[tdml] (
* directory as the tdml file, and some other variations.
*/
def findTDMLResource(resName: String): Option[URI] = {
- val resPath = Paths.get(resName)
- val resolvedURI =
- if (Files.exists(resPath)) Some(resPath.toFile().toURI())
- else Misc.getResourceRelativeOption(resName, Some(tsURI))
- val res = resolvedURI.orElse {
- // try ignoring the directory part
- val parts = resName.split("/")
- if (parts.length > 1) { // if there is one
- val filePart = parts.last
- val secondTry = findTDMLResource(filePart) // recursively
- secondTry
- } else {
- None
- }
- }
- res
+ Misc.searchResourceOption(resName, Some(tsURI))
}
def findEmbeddedSchema(modelName: String): Option[DefinedSchema] = {
@@ -869,6 +854,12 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
case t: Throwable => toss(t, implString)
}
+ // we should never need blobs if we're expecting an error even if we
+ // don't get errors. So clean them up immediately
+ actual.getBlobPaths.foreach { blobPath =>
+ Files.delete(blobPath)
+ }
+
val isErr: Boolean =
if (actual.isProcessingError) true
else {
@@ -1109,6 +1100,13 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
roundTrip match {
case NoRoundTrip => {
// done. Do nothing else.
+
+ // Done with the first parse result, safe to clean up blobs if there
+ // was success. This won't get called on failure, which is fine--leave
+ // blobs around for debugging
+ firstParseResult.getBlobPaths.foreach { blobPath =>
+ Files.delete(blobPath)
+ }
}
case OnePassRoundTrip => {
val outStream = new java.io.ByteArrayOutputStream()
@@ -1119,6 +1117,13 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
// directly back to the original input form.
VerifyTestCase.verifyUnparserTestData(new ByteArrayInputStream(firstParseTestData), outStream, implString)
+
+ // Done with the first parse result, safe to clean up blobs if there
+ // was success. This won't get called on failure, which is fine--leave
+ // blobs around for debugging
+ firstParseResult.getBlobPaths.foreach { blobPath =>
+ Files.delete(blobPath)
+ }
}
case TwoPassRoundTrip => {
//
@@ -1140,6 +1145,16 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
verifyParseResults(processor, actual, testInfoset, implString)
verifyLeftOverData(actual, reParseTestDataLength, implString)
// if it doesn't pass, it will throw out of here.
+
+ // Done with the first and second parse resultrs, safe to clean up
+ // blobs if there was success. This won't get called on failure, which
+ // is fine--leave blobs around for debugging
+ firstParseResult.getBlobPaths.foreach { blobPath =>
+ Files.delete(blobPath)
+ }
+ actual.getBlobPaths.foreach { blobPath =>
+ Files.delete(blobPath)
+ }
}
case ThreePassRoundTrip => {
//
@@ -1180,8 +1195,19 @@ case class ParserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
val thirdPassOutStream = new java.io.ByteArrayOutputStream()
doOnePassRoundTripUnparseExpectSuccess(processor, thirdPassOutStream, firstParseResult, implString)
VerifyTestCase.verifyUnparserTestData(new ByteArrayInputStream(reParseTestData), thirdPassOutStream, implString)
+
+ // Done with the first parse result and second parse results. Safe to
+ // clean up blobs if there was success. Leave them around for debugging
+ // if there was a failure
+ firstParseResult.getBlobPaths.foreach { blobPath =>
+ Files.delete(blobPath)
+ }
+ secondParseResult.getBlobPaths.foreach { blobPath =>
+ Files.delete(blobPath)
+ }
}
}
+
}
}
@@ -1325,6 +1351,14 @@ case class UnparserTestCase(ptc: NodeSeq, parentArg: DFDLTestSuite)
leftOverException.map {
throw _
} // if we get here, throw the left over data exception.
+
+ // Done with the parse results, safe to clean up blobs if there was
+ // success. This won't get called on failure, which is fine--leave blobs
+ // around for debugging
+ parseActual.getBlobPaths.foreach { blobPath =>
+ Files.delete(blobPath)
+ }
+
}
}
diff --git a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/processor/TDMLDFDLProcessor.scala b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/processor/TDMLDFDLProcessor.scala
index 1a0c2cd..3b5a908 100644
--- a/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/processor/TDMLDFDLProcessor.scala
+++ b/daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/processor/TDMLDFDLProcessor.scala
@@ -17,7 +17,12 @@
package org.apache.daffodil.tdml.processor
-import org.apache.daffodil.api.{ DaffodilSchemaSource, DataLocation, ValidationMode, Diagnostic }
+import java.nio.file.Path
+
+import org.apache.daffodil.api.DaffodilSchemaSource
+import org.apache.daffodil.api.DataLocation
+import org.apache.daffodil.api.Diagnostic
+import org.apache.daffodil.api.ValidationMode
import org.apache.daffodil.externalvars.Binding
object TDML {
@@ -85,6 +90,7 @@ trait TDMLResult {
trait TDMLParseResult extends TDMLResult {
def addDiagnostic(failure: Diagnostic): Unit
def getResult: scala.xml.Node
+ def getBlobPaths: Seq[Path] = Seq.empty
def currentLocation: DataLocation
}
diff --git a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/tdml/TDMLInfosetInputter.scala b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/tdml/TDMLInfosetInputter.scala
index b5a827d..3044b42 100644
--- a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/tdml/TDMLInfosetInputter.scala
+++ b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/tdml/TDMLInfosetInputter.scala
@@ -17,11 +17,14 @@
package org.apache.daffodil.tdml
+import java.net.URI
+
import org.apache.daffodil.infoset.InfosetInputter
import org.apache.daffodil.infoset.ScalaXMLInfosetInputter
import org.apache.daffodil.infoset.InfosetInputterEventType
import org.apache.daffodil.infoset.JsonInfosetInputter
import org.apache.daffodil.util.MaybeBoolean
+import org.apache.daffodil.util.Misc
import org.apache.daffodil.dpath.NodeInfo
class TDMLInfosetInputter(val scalaInputter: ScalaXMLInfosetInputter, others: Seq[InfosetInputter]) extends InfosetInputter {
@@ -83,7 +86,24 @@ class TDMLInfosetInputter(val scalaInputter: ScalaXMLInfosetInputter, others: Se
if (!othersmatch)
throw TDMLException("getSimpleText does not match", Some(implString))
- res
+
+ if (primType.isInstanceOf[NodeInfo.AnyURI.Kind]) {
+ val uri = new URI(res)
+ if (!uri.getPath.startsWith("/")) {
+ // TDML files must allow blob URI's to be relative, but Daffodil
+ // requires them to be absolute with a scheme. So search for the file
+ // using TDML semantics and convert to an absolute URI
+ val abs = Misc.searchResourceOption(uri.getPath, None)
+ if (abs.isEmpty) {
+ throw TDMLException("Unable to find URI: " + res, Some(implString))
+ }
+ abs.get.toString
+ } else {
+ res
+ }
+ } else {
+ res
+ }
}
override def isNilled(): MaybeBoolean = {
diff --git a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/tdml/processor/DaffodilTDMLDFDLProcessor.scala b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/tdml/processor/DaffodilTDMLDFDLProcessor.scala
index 683d9ee..73f4e15 100644
--- a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/tdml/processor/DaffodilTDMLDFDLProcessor.scala
+++ b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/tdml/processor/DaffodilTDMLDFDLProcessor.scala
@@ -18,24 +18,28 @@
package org.apache.daffodil.tdml.processor
import java.nio.channels.Channels
+import java.nio.file.Path
+import java.nio.file.Paths
+
+import scala.xml.Node
import org.apache.daffodil.api._
import org.apache.daffodil.compiler.Compiler
import org.apache.daffodil.debugger.Debugger
+import org.apache.daffodil.debugger.InteractiveDebugger
+import org.apache.daffodil.debugger.TraceDebuggerRunner
+import org.apache.daffodil.dsom.ExpressionCompilers
+import org.apache.daffodil.exceptions.Assert
import org.apache.daffodil.externalvars.Binding
import org.apache.daffodil.infoset.ScalaXMLInfosetInputter
import org.apache.daffodil.io.InputSourceDataInputStream
+import org.apache.daffodil.processors.DataProcessor
+import org.apache.daffodil.processors.UnparseResult
import org.apache.daffodil.processors.unparsers.UState
-import org.apache.daffodil.processors.{ DataProcessor, UnparseResult }
-import org.apache.daffodil.tdml.{ SchemaDataProcessorCache, TDMLInfosetOutputter }
-import org.apache.daffodil.util.MaybeULong
-
-import scala.xml.Node
-import org.apache.daffodil.debugger.InteractiveDebugger
import org.apache.daffodil.tdml.SchemaDataProcessorCache
-import org.apache.daffodil.debugger.TraceDebuggerRunner
-import org.apache.daffodil.dsom.ExpressionCompilers
-import org.apache.daffodil.exceptions.Assert
+import org.apache.daffodil.tdml.TDMLInfosetInputter
+import org.apache.daffodil.tdml.TDMLInfosetOutputter
+import org.apache.daffodil.util.MaybeULong
final class TDMLDFDLProcessorFactory() extends AbstractTDMLDFDLProcessorFactory {
@@ -130,6 +134,10 @@ class DaffodilTDMLDFDLProcessor(ddp: DFDL.DataProcessor) extends TDMLDFDLProcess
private lazy val builtInTracer = new InteractiveDebugger(new TraceDebuggerRunner, ExpressionCompilers)
+ private val blobDir = Paths.get(System.getProperty("java.io.tmpdir"), "daffodil-tdml", "blobs")
+ private val blobPrefix = ""
+ private val blobSuffix = ".bin"
+
override def setDebugging(b: Boolean) = dp.setDebugging(b)
override def setTracing(bool: Boolean): Unit = {
@@ -154,7 +162,9 @@ class DaffodilTDMLDFDLProcessor(ddp: DFDL.DataProcessor) extends TDMLDFDLProcess
override def getDiagnostics: Seq[Diagnostic] = dp.getDiagnostics
override def parse(is: java.io.InputStream, lengthLimitInBits: Long): TDMLParseResult = {
+
val outputter = new TDMLInfosetOutputter()
+ outputter.setBlobAttributes(blobDir, blobPrefix, blobSuffix)
val dis = InputSourceDataInputStream(is)
if (lengthLimitInBits >= 0 && lengthLimitInBits % 8 != 0) {
@@ -171,7 +181,12 @@ class DaffodilTDMLDFDLProcessor(ddp: DFDL.DataProcessor) extends TDMLDFDLProcess
override def unparse(infosetXML: scala.xml.Node, outStream: java.io.OutputStream): TDMLUnparseResult = {
val output = java.nio.channels.Channels.newChannel(outStream)
- val inputter = new ScalaXMLInfosetInputter(infosetXML)
+ val scalaInputter = new ScalaXMLInfosetInputter(infosetXML)
+ // We can't compare against other inputters since we only have scala XML,
+ // but we still need to use the TDMLInfosetInputter since it may make TDML
+ // specific modifications to the input infoset (e.g. blob paths)
+ val otherInputters = Seq.empty
+ val inputter = new TDMLInfosetInputter(scalaInputter, otherInputters)
val actual = dp.unparse(inputter, output).asInstanceOf[UnparseResult]
output.close()
new DaffodilTDMLUnparseResult(actual, outStream)
@@ -192,6 +207,8 @@ final class DaffodilTDMLParseResult(actual: DFDL.ParseResult, outputter: TDMLInf
override def getResult: Node = outputter.getResult()
+ override def getBlobPaths: Seq[Path] = outputter.getBlobPaths()
+
def inputter = outputter.toInfosetInputter()
override def isProcessingError: Boolean = actual.isProcessingError
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/Blobs.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/Blobs.tdml
new file mode 100644
index 0000000..a9f96c4
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/Blobs.tdml
@@ -0,0 +1,349 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<tdml:testSuite suiteName="SimpleTypes" description="Section 5 - simple types"
+ xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext"
+ xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"
+ xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions"
+ xmlns:ex="http://example.com"
+ xmlns:tdml="http://www.ibm.com/xmlns/dfdl/testData"
+ xmlns:tns="http://example.com"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ defaultRoundTrip="onePass"
+ defaultConfig="blobChunkSize">
+
+ <!-- set the blob chunking size to 1 hopefully catch edge cases -->
+ <tdml:defineConfig name="blobChunkSize">
+ <daf:tunables>
+ <daf:blobChunkSizeInBytes>1</daf:blobChunkSizeInBytes>
+ </daf:tunables>
+ </tdml:defineConfig>
+
+
+ <tdml:defineSchema name="Blob.dfdl.xsd">
+
+ <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/>
+
+ <dfdl:format ref="ex:GeneralFormatPortable"
+ representation="binary"
+ encodingErrorPolicy="replace" />
+
+ <xs:element name="blob_01" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="4"
+ dfdlx:objectKind="bytes" />
+
+ <xs:element name="blob_02" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="30" dfdl:lengthUnits="bits"
+ dfdl:bitOrder="mostSignificantBitFirst"
+ dfdlx:objectKind="bytes" />
+
+ <xs:element name="blob_03" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="30" dfdl:lengthUnits="bits"
+ dfdl:bitOrder="leastSignificantBitFirst" dfdl:byteOrder="littleEndian"
+ dfdlx:objectKind="bytes" />
+
+ <xs:element name="blob_04">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="length" type="xs:int" dfdl:lengthKind="explicit" dfdl:length="4"
+ dfdl:outputValueCalc="{ dfdl:valueLength(../ex:data, 'bytes') }" />
+ <xs:element name="data" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="{ ../ex:length }"
+ dfdlx:objectKind="bytes" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="blob_05">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="length" type="xs:int" dfdl:lengthKind="explicit" dfdl:length="4"
+ dfdl:outputValueCalc="{ dfdl:valueLength(../ex:data, 'bytes') }" />
+ <xs:element name="data" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="{ ../ex:length }"
+ dfdlx:objectKind="bytes" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="blob_06" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="4" dfdl:lengthUnits="characters"
+ dfdlx:objectKind="bytes" />
+
+ <xs:element name="blob_07" type="xs:anyURI"
+ dfdl:lengthKind="implicit"
+ dfdlx:objectKind="bytes" />
+
+ <xs:element name="blob_08">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="a" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="1"
+ dfdlx:objectKind="bytes" />
+ <xs:choice>
+ <xs:sequence>
+ <xs:element name="b" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="2"
+ dfdlx:objectKind="bytes" />
+ <xs:element name="c" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="2"
+ dfdlx:objectKind="bytes" />
+ </xs:sequence>
+ <xs:element name="d" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="3"
+ dfdlx:objectKind="bytes" />
+ </xs:choice>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="blob_09">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="a" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="1"
+ dfdlx:objectKind="bytes" />
+ <xs:element name="len" type="xs:int" dfdl:lengthKind="explicit" dfdl:length="1" />
+ <xs:element name="b" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="{ ../ex:len }"
+ dfdlx:objectKind="bytes" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="clob_01" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="4"
+ dfdlx:objectKind="chars" />
+
+ </tdml:defineSchema>
+
+
+ <!-- 4 bytes of blob data -->
+ <tdml:parserTestCase name="blob_01" root="blob_01" model="Blob.dfdl.xsd">
+ <tdml:document>
+ <tdml:documentPart type="byte"><![CDATA[a1b1c1d1]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:blob_01 xsi:type="xs:anyURI">org/apache/daffodil/section05/simple_types/blobs/blob_01.bin</ex:blob_01>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
+
+ <!-- 30 bits of blob data, MSBF -->
+ <tdml:parserTestCase name="blob_02" root="blob_02" model="Blob.dfdl.xsd">
+ <tdml:document>
+ <tdml:documentPart type="bits"><![CDATA[11011110 10101101 10111110 111011xx]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:blob_02 xsi:type="xs:anyURI">org/apache/daffodil/section05/simple_types/blobs/blob_02.bin</ex:blob_02>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
+
+ <!-- 30 bits of blob data, LSBF -->
+ <tdml:parserTestCase name="blob_03" root="blob_03" model="Blob.dfdl.xsd">
+ <tdml:document bitOrder="LSBFirst">
+ <tdml:documentPart type="bits" byteOrder="RTL"><![CDATA[xx101111 10111110 10101101 11011110]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:blob_03 xsi:type="xs:anyURI">org/apache/daffodil/section05/simple_types/blobs/blob_03.bin</ex:blob_03>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
+
+ <!-- unparse 3 bytes of blob data with 1 fill byte -->
+ <tdml:unparserTestCase name="blob_04" root="blob_01" model="Blob.dfdl.xsd" roundTrip="none">
+ <tdml:document>
+ <tdml:documentPart type="byte"><![CDATA[a1b1c120]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:blob_01 xsi:type="xs:anyURI">org/apache/daffodil/section05/simple_types/blobs/blob_04.bin</ex:blob_01>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:unparserTestCase>
+
+ <!-- dynamic number of blob data, parse -->
+ <tdml:parserTestCase name="blob_05" root="blob_04" model="Blob.dfdl.xsd">
+ <tdml:document>
+ <tdml:documentPart type="byte"><![CDATA[00 00 00 04 a1 b1 c1 d1]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:blob_04>
+ <ex:length>4</ex:length>
+ <ex:data xsi:type="xs:anyURI">org/apache/daffodil/section05/simple_types/blobs/blob_01.bin</ex:data>
+ </ex:blob_04>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
+
+ <!-- dynamic number of blob data, unparse -->
+ <tdml:unparserTestCase name="blob_06" root="blob_04" model="Blob.dfdl.xsd" roundTrip="false">
+ <tdml:document>
+ <tdml:documentPart type="byte"><![CDATA[00 00 00 04 a1 b1 c1 d1]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:blob_04>
+ <ex:length>0</ex:length>
+ <ex:data xsi:type="xs:anyURI">org/apache/daffodil/section05/simple_types/blobs/blob_01.bin</ex:data>
+ </ex:blob_04>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:unparserTestCase>
+
+ <!-- unparsed data longer that length -->
+ <tdml:unparserTestCase name="blob_07" root="blob_01" model="Blob.dfdl.xsd">
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:blob_01 xsi:type="xs:anyURI">org/apache/daffodil/section05/simple_types/blobs/blob_07.bin</ex:blob_01>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ <tdml:errors>
+ <tdml:error>Unparse Error</tdml:error>
+ <tdml:error>Blob length</tdml:error>
+ <tdml:error>40 bits</tdml:error>
+ <tdml:error>exceeds</tdml:error>
+ <tdml:error>32 bits</tdml:error>
+ </tdml:errors>
+ </tdml:unparserTestCase>
+
+ <!-- unparse, blob file not found -->
+ <tdml:unparserTestCase name="blob_08" root="blob_01" model="Blob.dfdl.xsd">
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:blob_01 xsi:type="xs:anyURI">file:/this/file/does/not/exist</ex:blob_01>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ <tdml:errors>
+ <tdml:error>Unparse Error</tdml:error>
+ <tdml:error>Unable to open blob for reading</tdml:error>
+ <tdml:error>file:/this/file/does/not/exist</tdml:error>
+ </tdml:errors>
+ </tdml:unparserTestCase>
+
+ <!-- unparse, blob must be a file -->
+ <tdml:unparserTestCase name="blob_09" root="blob_01" model="Blob.dfdl.xsd">
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:blob_01 xsi:type="xs:anyURI">http://scheme/not/allowed</ex:blob_01>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ <tdml:errors>
+ <tdml:error>Unparse Error</tdml:error>
+ <tdml:error>Blob URI must be a file</tdml:error>
+ <tdml:error>http://scheme/not/allowed</tdml:error>
+ </tdml:errors>
+ </tdml:unparserTestCase>
+
+ <!-- parse, not enough bits -->
+ <tdml:parserTestCase name="blob_10" root="blob_01" model="Blob.dfdl.xsd">
+ <tdml:document>
+ <tdml:documentPart type="byte"><![CDATA[a1b1c1]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:errors>
+ <tdml:error>Parse Error</tdml:error>
+ <tdml:error>Insufficient bits in data</tdml:error>
+ <tdml:error>Needed 32 bit(s)</tdml:error>
+ </tdml:errors>
+ </tdml:parserTestCase>
+
+ <!-- sde, invalid lengthUnits -->
+ <tdml:parserTestCase name="blob_11" root="blob_06" model="Blob.dfdl.xsd">
+ <tdml:document>
+ <tdml:documentPart type="byte"><![CDATA[a1b1c1]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:errors>
+ <tdml:error>Schema Definition Error</tdml:error>
+ <tdml:error>lengthKind 'explicit'</tdml:error>
+ <tdml:error>lengthUnits 'characters'</tdml:error>
+ </tdml:errors>
+ </tdml:parserTestCase>
+
+ <!-- sde, invalid lengthKind -->
+ <tdml:parserTestCase name="blob_12" root="blob_07" model="Blob.dfdl.xsd">
+ <tdml:document>
+ <tdml:documentPart type="byte"><![CDATA[a1b1c1]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:errors>
+ <tdml:error>Schema Definition Error</tdml:error>
+ <tdml:error>objectKind='bytes'</tdml:error>
+ <tdml:error>must have dfdl:lengthKind='explicit'</tdml:error>
+ </tdml:errors>
+ </tdml:parserTestCase>
+
+ <!-- 4 bytes of blob data split in multiple elements with a choice, backtracking should remove blobs -->
+ <tdml:parserTestCase name="blob_13" root="blob_08" model="Blob.dfdl.xsd">
+ <tdml:document>
+ <tdml:documentPart type="byte"><![CDATA[a1b1c1d1]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:blob_08>
+ <ex:a xsi:type="xs:anyURI">org/apache/daffodil/section05/simple_types/blobs/blob_13a.bin</ex:a>
+ <ex:d xsi:type="xs:anyURI">org/apache/daffodil/section05/simple_types/blobs/blob_13d.bin</ex:d>
+ </ex:blob_08>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
+
+ <!-- 4 bytes of blob data split in multiple elements with a choice, no backtracking needed -->
+ <tdml:parserTestCase name="blob_14" root="blob_08" model="Blob.dfdl.xsd">
+ <tdml:document>
+ <tdml:documentPart type="byte"><![CDATA[a1b1c1d1e1]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:blob_08>
+ <ex:a xsi:type="xs:anyURI">org/apache/daffodil/section05/simple_types/blobs/blob_13a.bin</ex:a>
+ <ex:b xsi:type="xs:anyURI">org/apache/daffodil/section05/simple_types/blobs/blob_13b.bin</ex:b>
+ <ex:c xsi:type="xs:anyURI">org/apache/daffodil/section05/simple_types/blobs/blob_13c.bin</ex:c>
+ </ex:blob_08>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
+
+ <!-- 1 byte of blob, length, 3 bytes of blob, but invalid len caused SDE, blob should all be deleted -->
+ <tdml:parserTestCase name="blob_15" root="blob_09" model="Blob.dfdl.xsd">
+ <tdml:document>
+ <tdml:documentPart type="byte"><![CDATA[a1ffb1c1]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:errors>
+ <tdml:error>Runtime Schema Definition Error</tdml:error>
+ <tdml:error>dfdl:length expression result must be non-negative</tdml:error>
+ </tdml:errors>
+ </tdml:parserTestCase>
+
+ <!-- clob unsupported -->
+ <tdml:parserTestCase name="clob_01" root="clob_01" model="Blob.dfdl.xsd">
+ <tdml:document>
+ <tdml:documentPart type="text">clobdata</tdml:documentPart>
+ </tdml:document>
+ <tdml:errors>
+ <tdml:error>Schema Definition Error</tdml:error>
+ <tdml:error>objectKind='chars' is not supported.</tdml:error>
+ </tdml:errors>
+ </tdml:parserTestCase>
+
+
+</tdml:testSuite>
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/SimpleTypes.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/SimpleTypes.tdml
index 99d0213..d1779a2 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/SimpleTypes.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/SimpleTypes.tdml
@@ -19,6 +19,7 @@
<tdml:testSuite suiteName="SimpleTypes" description="Section 5 - simple types"
xmlns:tdml="http://www.ibm.com/xmlns/dfdl/testData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/" xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions"
xmlns:ex="http://example.com" xmlns:tns="http://example.com"
defaultRoundTrip="true">
@@ -3997,7 +3998,7 @@
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:parserTestCase>
-
+
<!--
Test name: hexBinary_rep
Schema: HexBinary
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_01.bin b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_01.bin
new file mode 100644
index 0000000..c6e0510
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_01.bin
@@ -0,0 +1 @@
+����
\ No newline at end of file
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_02.bin b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_02.bin
new file mode 100644
index 0000000..01904d7
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_02.bin
@@ -0,0 +1 @@
+ޭ��
\ No newline at end of file
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_03.bin b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_03.bin
new file mode 100644
index 0000000..39236d8
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_03.bin
@@ -0,0 +1 @@
+ޭ�/
\ No newline at end of file
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_04.bin b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_04.bin
new file mode 100644
index 0000000..ad90cb5
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_04.bin
@@ -0,0 +1 @@
+���
\ No newline at end of file
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_07.bin b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_07.bin
new file mode 100644
index 0000000..d26a6fe
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_07.bin
@@ -0,0 +1 @@
+�����
\ No newline at end of file
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_13a.bin b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_13a.bin
new file mode 100644
index 0000000..57814dd
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_13a.bin
@@ -0,0 +1 @@
+�
\ No newline at end of file
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_13b.bin b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_13b.bin
new file mode 100644
index 0000000..537a638
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_13b.bin
@@ -0,0 +1 @@
+��
\ No newline at end of file
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_13c.bin b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_13c.bin
new file mode 100644
index 0000000..3ad68b7
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_13c.bin
@@ -0,0 +1 @@
+��
\ No newline at end of file
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_13d.bin b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_13d.bin
new file mode 100644
index 0000000..2a5e62b
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/blobs/blob_13d.bin
@@ -0,0 +1 @@
+���
\ No newline at end of file
diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section05/simple_types/TestBlobs.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section05/simple_types/TestBlobs.scala
new file mode 100644
index 0000000..4438043
--- /dev/null
+++ b/daffodil-test/src/test/scala/org/apache/daffodil/section05/simple_types/TestBlobs.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.section05.simple_types
+
+import org.junit._
+import org.apache.daffodil.tdml.Runner
+import org.junit.AfterClass
+
+object TestBlobs {
+
+ val testDir = "/org/apache/daffodil/section05/simple_types/"
+
+ val runner = Runner(testDir, "Blobs.tdml")
+
+ @AfterClass def shutDown() {
+ runner.reset
+ }
+}
+
+class TestBlobs {
+ import TestBlobs._
+
+ @Test def test_blob_01() { runner.runOneTest("blob_01") }
+ @Test def test_blob_02() { runner.runOneTest("blob_02") }
+ @Test def test_blob_03() { runner.runOneTest("blob_03") }
+ @Test def test_blob_04() { runner.runOneTest("blob_04") }
+ @Test def test_blob_05() { runner.runOneTest("blob_05") }
+ @Test def test_blob_06() { runner.runOneTest("blob_06") }
+ @Test def test_blob_07() { runner.runOneTest("blob_07") }
+ @Test def test_blob_08() { runner.runOneTest("blob_08") }
+ @Test def test_blob_09() { runner.runOneTest("blob_09") }
+ @Test def test_blob_10() { runner.runOneTest("blob_10") }
+ @Test def test_blob_11() { runner.runOneTest("blob_11") }
+ @Test def test_blob_12() { runner.runOneTest("blob_12") }
+ @Test def test_blob_13() { runner.runOneTest("blob_13") }
+ @Test def test_blob_14() { runner.runOneTest("blob_14") }
+ @Test def test_blob_15() { runner.runOneTest("blob_15") }
+
+ @Test def test_clob_01() { runner.runOneTest("clob_01") }
+
+}