You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by ja...@apache.org on 2019/12/09 21:54:41 UTC
[incubator-daffodil] branch master updated: Switch to file output
stream for large data
This is an automated email from the ASF dual-hosted git repository.
jadams pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git
The following commit(s) were added to refs/heads/master by this push:
new afc9033 Switch to file output stream for large data
afc9033 is described below
commit afc903330bc91bd2125ba2b5c7449ec45a2cdb69
Author: Josh Adams <ja...@tresys.com>
AuthorDate: Mon Nov 11 12:45:15 2019 -0500
Switch to file output stream for large data
Previously, we exclusively were using a ByteArrayOutputStream to handle
buffered output streams, which has a hard limit of 2GB. This commit
allows the output stream to change to a FileOutputStream to a temporary
file once the buffer grows to a specified size, removing the 2GB limit.
This change also removes the BufferedBlobDataOutputStream as it is not
longer needed.
DAFFODIL-2194
---
.../org/apache/daffodil/io/DataOutputStream.scala | 28 +-
.../daffodil/io/DataOutputStreamImplMixin.scala | 79 +-
.../io/DirectOrBufferedDataOutputStream.scala | 283 +++++---
.../apache/daffodil/io/TestDataOutputStream.scala | 56 +-
.../apache/daffodil/io/TestDataOutputStream2.scala | 14 +-
.../apache/daffodil/io/TestDataOutputStream3.scala | 56 +-
.../apache/daffodil/io/TestDataOutputStream4.scala | 8 +-
.../io/TestDirectOrBufferedDataOutputStream.scala | 14 +-
.../resources/org/apache/daffodil/xsd/dafext.xsd | 26 +
.../apache/daffodil/propGen/TunableGenerator.scala | 8 +
.../processors/unparsers/BlobLengthUnparser.scala | 4 +-
.../apache/daffodil/layers/LayerTransformer.scala | 8 +-
.../apache/daffodil/processors/DataProcessor.scala | 18 +-
.../daffodil/processors/unparsers/UState.scala | 9 +-
.../general/testUnparserFileBuffering.tdml | 798 +++++++++++++++++++++
.../section00/general/TestFileBuffering.scala | 68 ++
16 files changed, 1290 insertions(+), 187 deletions(-)
diff --git a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStream.scala b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStream.scala
index 4518b19..4d50a0f 100644
--- a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStream.scala
+++ b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStream.scala
@@ -19,6 +19,8 @@ package org.apache.daffodil.io
import passera.unsigned.ULong
import java.nio.CharBuffer
+import java.nio.file.Path
+import java.io.File
import org.apache.daffodil.util.MaybeULong
import org.apache.daffodil.util.Logging
import org.apache.daffodil.util.Maybe
@@ -97,6 +99,20 @@ trait DataOutputStream extends DataStreamCommon
def maybeAbsBitPos0b: MaybeULong
/**
+ * These values are used for output streams that could change from
+ * ByteArray's to File based output streams.
+ */
+ def chunkSizeInBytes: Int
+ def maxBufferSizeInBytes: Long
+ def tempDirPath: File
+
+ /**
+ * maybeExistingFile is used in the case of blob files, where we already have an
+ * existing file containing the data. This is the path to said file.
+ */
+ def maybeExistingFile: Maybe[Path]
+
+ /**
* Besides setting the relBitPos, it also maintains the value of
* the absolute bit pos, if it is known.
*/
@@ -171,10 +187,15 @@ trait DataOutputStream extends DataStreamCommon
*
* If not enough bits are available, this writes nothing and returns false.
*
+ * ignoreByteOrder is used when we are working with a FileOutputStream. Given
+ * that the bytes are already converted to BigEndian when they are written out
+ * to a file, we no longer need to convert little endian arrays so byte order
+ * can be ignored.
+ *
* It is a usage error if bitLengthFrom1 is not greater than or equal to 1.
*
*/
- def putByteArray(ba: Array[Byte], bitLengthFrom1: Int, finfo: FormatInfo): Boolean
+ def putByteArray(ba: Array[Byte], bitLengthFrom1: Int, finfo: FormatInfo, ignoreByteOrder: Boolean = false): Boolean
/**
* Float and Double
@@ -199,5 +220,10 @@ trait DataOutputStream extends DataStreamCommon
def setFinished(finfo: FormatInfo): Unit
def isFinished: Boolean
+ /**
+ * This function deletes any temnporary files that have been generated
+ */
+ def cleanUp(): Unit
+
def zeroLengthStatus: ZeroLengthStatus
}
diff --git a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStreamImplMixin.scala b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStreamImplMixin.scala
index a5d1b2d..6b9babe 100644
--- a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStreamImplMixin.scala
+++ b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStreamImplMixin.scala
@@ -23,6 +23,9 @@ import org.apache.daffodil.util.Maybe.Nope
import org.apache.daffodil.schema.annotation.props.gen.ByteOrder
import org.apache.daffodil.schema.annotation.props.gen.BitOrder
import passera.unsigned.ULong
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
import java.nio.ByteBuffer
import org.apache.daffodil.exceptions.Assert
import java.nio.charset.CoderResult
@@ -35,6 +38,8 @@ import org.apache.daffodil.util.Bits
import org.apache.daffodil.util.LogLevel
import org.apache.daffodil.processors.charset.BitsCharsetNonByteSizeEncoder
+import java.util.Arrays
+
sealed trait DOSState
private[io] case object Active extends DOSState
private[io] case object Finished extends DOSState
@@ -465,7 +470,11 @@ trait DataOutputStreamImplMixin extends DataStreamCommonState
res
}
- final override def putByteArray(array: Array[Byte], bitLengthFrom1: Int, finfo: FormatInfo): Boolean = {
+ final override def putByteArray(
+ array: Array[Byte],
+ bitLengthFrom1: Int,
+ finfo: FormatInfo,
+ ignoreByteOrder: Boolean = false): Boolean = {
// this is to be used for an array generated by getByteArray. Thus, this
// array is expected by to BigEndian MSBF. It must be transformed into an
// array that the other putBytes/Bits/etc functions can accept
@@ -485,7 +494,9 @@ trait DataOutputStreamImplMixin extends DataStreamCommonState
// byte sizes are common and it avoids the byteOrder check and
// potential array allocation
val bytes =
- if (bitLengthFrom1 > 8 && finfo.byteOrder =:= ByteOrder.LittleEndian) {
+ if (bitLengthFrom1 > 8 &&
+ finfo.byteOrder =:= ByteOrder.LittleEndian &&
+ !ignoreByteOrder) {
// We need to reverse this array. However, we cannot modify this
// array since it might come straight from the infoset, which could
// potentiall affect expressions that reference this data. So
@@ -657,6 +668,62 @@ trait DataOutputStreamImplMixin extends DataStreamCommonState
}
}
+ final def putFile(
+ path: Path,
+ lengthInBits: Long,
+ chunkSizeInBytes: Int,
+ finfo: FormatInfo): Long = {
+
+ val fileStream =
+ try {
+ Files.newInputStream(path, StandardOpenOption.READ)
+ } catch {
+ case e: Exception =>
+ throw new FileIOException(
+ "Unable to open file %s for reading: %s".format(
+ path.toString,
+ e.getMessage()))
+ }
+
+ val array = new Array[Byte](chunkSizeInBytes)
+
+ val chunkSizeInBits = chunkSizeInBytes * 8
+
+ var remainingBitsToPut = lengthInBits
+ var fileHasData = true
+ var bitsWritten: Long = 0
+ while (remainingBitsToPut > 0 && fileHasData) {
+ val bitsToRead = Math.min(remainingBitsToPut, chunkSizeInBits)
+ val bytesToRead = (bitsToRead + 7) / 8
+ val bytesRead = fileStream.read(array, 0, bytesToRead.toInt)
+ if (bytesRead == -1) {
+ fileHasData = false
+ } else {
+ val bitsToPut = Math.min(bytesRead * 8, bitsToRead)
+ val ret = putByteArray(array, bitsToPut.toInt, finfo, ignoreByteOrder=true)
+ if (!ret) {
+ fileStream.close()
+ throw new FileIOException(
+ "Failed to write file data: %s".format(path.toString))
+ }
+ bitsWritten += bitsToPut
+ remainingBitsToPut -= bitsToPut
+ }
+ }
+
+ fileStream.close()
+
+ // calculate the skip bits
+ val nFillBits = remainingBitsToPut
+ if (nFillBits > 0) {
+ val ret = skip(nFillBits, finfo)
+ Assert.invariant(ret)
+ bitsWritten += nFillBits
+ }
+ bitsWritten
+ }
+
+
/**
* Returns number of bytes transferred. Stops when the bitLimit is
* encountered if one is defined.
@@ -820,7 +887,11 @@ trait DataOutputStreamImplMixin extends DataStreamCommonState
*
* Assumed to be called from inner loops, so should be fast as possible.
*/
- protected final def putLongUnchecked(signedLong: Long, bitLengthFrom1To64: Int, finfo: FormatInfo): Boolean = {
+ protected final def putLongUnchecked(
+ signedLong: Long,
+ bitLengthFrom1To64: Int,
+ finfo: FormatInfo,
+ ignoreByteOrder: Boolean = false): Boolean = {
// The order we check bitOrder vs byteOrder here is actually very
// important, primarily due to how suspensions end up resolved and data
// output streams (DOS) collapsed. To explain why:
@@ -853,7 +924,7 @@ trait DataOutputStreamImplMixin extends DataStreamCommonState
// the only time that combination can legally occurs is in this edge case
// of collapsing a fragment byte, and then byte order doesn't matter.
val res =
- if (finfo.bitOrder eq BitOrder.MostSignificantBitFirst) {
+ if ((finfo.bitOrder eq BitOrder.MostSignificantBitFirst) && !ignoreByteOrder) {
if (finfo.byteOrder eq ByteOrder.BigEndian) {
putLong_BE_MSBFirst(signedLong, bitLengthFrom1To64)
} else {
diff --git a/daffodil-io/src/main/scala/org/apache/daffodil/io/DirectOrBufferedDataOutputStream.scala b/daffodil-io/src/main/scala/org/apache/daffodil/io/DirectOrBufferedDataOutputStream.scala
index 93caee4..30f53cd 100644
--- a/daffodil-io/src/main/scala/org/apache/daffodil/io/DirectOrBufferedDataOutputStream.scala
+++ b/daffodil-io/src/main/scala/org/apache/daffodil/io/DirectOrBufferedDataOutputStream.scala
@@ -33,32 +33,98 @@ import org.apache.daffodil.util.Maybe
import org.apache.daffodil.util.Maybe._
import org.apache.daffodil.util.MaybeULong
import org.apache.daffodil.util.Misc
+import org.apache.daffodil.util.Logging
+import java.io.File
/**
* This simple extension just gives us a public method for access to the underlying byte array.
* That way we don't have to make a copy just to access the bytes.
*/
private[io] class ByteArrayOutputStreamWithGetBuf() extends java.io.ByteArrayOutputStream {
- def getBuf() = buf
- def getCount() = count
+ def getBuf = buf
+}
- def toDebugContent = {
- val content = toString("iso-8859-1")
- val s = Misc.remapControlsAndLineEndingsToVisibleGlyphs(content)
- s
+private[io] class ByteArrayOrFileOutputStream(
+ maxBufferSizeInBytes: Long,
+ tempDirPath: File,
+ maybeExistingFile: Maybe[Path])
+ extends java.io.OutputStream with Logging {
+
+ var isFile: Boolean = maybeExistingFile.isDefined
+
+ // This will be the file object that can represent either a generated
+ // temporary file or an existing blob file
+ var maybeFile: Maybe[File] = {
+ if (maybeExistingFile.isDefined)
+ Maybe(maybeExistingFile.get.toFile)
+ else
+ Maybe.Nope
}
- override def reset(): Unit = {
- var i: Int = 0 // Performance. This clearing isn't necessary. Just makes debug easier.
- while (i < count) {
- buf(i) = 0
- i += 1
- }
- super.reset()
+ // Keep track of whether or not this is a temporary file. If it is an existing
+ // file, i.e. a blob file, we do not want to delete it as that is handled
+ // elsewhere.
+ lazy val isTempFile = (!maybeExistingFile.isDefined && isFile)
+
+ // This is only used to determine if we need to switch to a file based output
+ // stream
+ var nBytes: Long = 0
+ var stream: java.io.OutputStream = new ByteArrayOutputStreamWithGetBuf()
+
+ /**
+ * Check to see if there is enough room in the ByteArrayOutputStream for the
+ * specified length. If there is not, switch to FileOutputStream.
+ */
+ @inline
+ private def checkBuffer(lengthInBytes: Long): Unit = {
+ if (!isFile && (nBytes + lengthInBytes > maxBufferSizeInBytes)) {
+ log(LogLevel.Info, "Switching to file based output stream. If this is performance critical, you may want to consider re-organizing your schema to avoid this if possible.")
+ maybeFile = try {
+ val file = File.createTempFile("daffodil-", ".tmp", tempDirPath)
+ file.deleteOnExit()
+ Maybe(file)
+ } catch {
+ case e: Exception =>
+ throw new FileIOException("Unable to create temporary file in %s: %s".format(tempDirPath.getPath, e.getMessage()))
+ }
+ val newStream: java.io.OutputStream = new java.io.FileOutputStream(maybeFile.get)
+ stream.flush
+ stream.asInstanceOf[ByteArrayOutputStreamWithGetBuf].writeTo(newStream)
+ stream = newStream
+ isFile = true
+ } else
+ nBytes += lengthInBytes
+ }
+
+ override def write(b: Array[Byte]) = {
+ checkBuffer(b.length)
+ stream.write(b)
}
- def hexDump = {
- (0 to (count - 1)).map { i => "%2x".format(buf(i).toInt & 0xFF) }.mkString(".")
+ override def write(b: Array[Byte], off: Int, len: Int) = {
+ checkBuffer(len)
+ stream.write(b, off, len)
+ }
+
+ override def write(b: Int) = {
+ checkBuffer(1)
+ stream.write(b)
+ }
+
+ override def close() = stream.close()
+
+ def getBuf() = {
+ Assert.usage(!isFile, "Attempted to call getBuf on FileOutputStream")
+ stream.asInstanceOf[ByteArrayOutputStreamWithGetBuf].getBuf
+ }
+
+ def getFile: File = maybeFile.get
+
+ def hexDump: String = {
+ if (isFile)
+ maybeFile.get.toString
+ else
+ (0 to (nBytes.toInt - 1)).map { i => "%2x".format(getBuf()(i).toInt & 0xFF) }.mkString(".")
}
override def toString = hexDump
@@ -93,8 +159,28 @@ private[io] class ByteArrayOutputStreamWithGetBuf() extends java.io.ByteArrayOut
* is not related to a layer, that means the associated Java OutputStream came
* from the user and it is the users responsibility to close it. The isLayer
* provides the flag to know which streams should be closed or not.
+ *
+ * chunkSizeInBytes is used when the buffered output stream is using a file as
+ * its buffer. This is the size of chunks that will be read into memory before
+ * being written to the direct output stream.
+ *
+ * maxBufferSizeInByte is the size that the ByteArrayOutputStream will grow to
+ * before switching over to a FileOutputStream
+ *
+ * tempDirPath is the path where temporary files will be created when switching
+ * to a file based buffer
+ *
+ * maybeExistingFile is used in the case of blob files, where we already have an
+ * existing file containing the data. This is the path to said file.
+ *
*/
-class DirectOrBufferedDataOutputStream private[io] (var splitFrom: DirectOrBufferedDataOutputStream, val isLayer: Boolean = false)
+class DirectOrBufferedDataOutputStream private[io] (
+ var splitFrom: DirectOrBufferedDataOutputStream,
+ val isLayer: Boolean = false,
+ val chunkSizeInBytes: Int,
+ val maxBufferSizeInBytes: Long,
+ val tempDirPath: File,
+ val maybeExistingFile: Maybe[Path])
extends DataOutputStreamImplMixin {
type ThisType = DirectOrBufferedDataOutputStream
@@ -183,7 +269,7 @@ class DirectOrBufferedDataOutputStream private[io] (var splitFrom: DirectOrBuffe
*
* If reused, this must be reset.
*/
- protected val bufferingJOS = new ByteArrayOutputStreamWithGetBuf()
+ protected val bufferingJOS = new ByteArrayOrFileOutputStream(maxBufferSizeInBytes, tempDirPath, maybeExistingFile)
/**
* Switched to point a either the buffering or direct java output stream in order
@@ -225,7 +311,13 @@ class DirectOrBufferedDataOutputStream private[io] (var splitFrom: DirectOrBuffe
* be completely configured (byteOrder, encoding, bitOrder, etc.)
*/
def addBuffered(): DirectOrBufferedDataOutputStream = {
- val buffered = new DirectOrBufferedDataOutputStream(this, isLayer)
+ val buffered = new DirectOrBufferedDataOutputStream(
+ this,
+ isLayer,
+ chunkSizeInBytes,
+ maxBufferSizeInBytes,
+ tempDirPath,
+ Maybe.Nope)
addBufferedDOS(buffered)
buffered
}
@@ -240,12 +332,13 @@ class DirectOrBufferedDataOutputStream private[io] (var splitFrom: DirectOrBuffe
// DOS to it. When this normal DOS is finished, the blob DOS will handle
// delivering the blob data to it without loading the whole blob into
// memory all at once.
- val bufferedBlob = new BufferedBlobDataOutputStream(
+ val bufferedBlob = new DirectOrBufferedDataOutputStream(
this,
- path,
- lengthInBits,
+ isLayer,
blobChunkSizeInBytes,
- isLayer)
+ 0,
+ tempDirPath,
+ Maybe(path))
addBufferedDOS(bufferedBlob)
// we know the length of the blob as passed in, so adjust the bit position
@@ -407,6 +500,7 @@ class DirectOrBufferedDataOutputStream private[io] (var splitFrom: DirectOrBuffe
// of buffers.
//
setDOSState(Finished)
+ this.getJavaOutputStream().close()
if (_following.isDefined) {
val f = _following.get
@@ -757,23 +851,51 @@ class DirectOrBufferedDataOutputStream private[io] (var splitFrom: DirectOrBuffe
// private def withBitLengthLimit(lengthLimitInBits: Long)(body: => Unit): Boolean = macro IOMacros.withBitLengthLimitMacroForOutput
protected def deliverContent(directDOS: DirectOrBufferedDataOutputStream, finfo: FormatInfo) = {
-
- val ba = this.bufferingJOS.getBuf
val bufferNBits = this.relBitPos0b // don't have to subtract a starting offset. It's always zero in buffered case.
+ val bufOS = this.bufferingJOS
if (directDOS.isEndOnByteBoundary && this.isEndOnByteBoundary) {
// no fragment bytes anywhere - just take the bytes
val nBytes = (bufferNBits / 8).toInt
- val nBytesPut = directDOS.putBytes(ba, 0, nBytes, finfo)
+ val nBytesPut = {
+ if (bufOS.isFile) {
+ bufOS.close
+ val nBitsPut = try {
+ directDOS.putFile(bufOS.getFile.toPath, bufferNBits.toLong, chunkSizeInBytes, finfo)
+ } finally {
+ // Make sure we delete the file after it was put in the directDOS or
+ // if we encountered an error
+ if (bufOS.isTempFile)
+ bufOS.getFile.delete()
+ }
+ nBitsPut / 8
+ } else
+ directDOS.putBytes(bufOS.getBuf, 0, nBytes, finfo)
+ }
Assert.invariant(nBytesPut == nBytes)
} else {
// fragment byte on directDOS, fragment byte on bufDOS, or both.
val nFragBits = this.fragmentLastByteLimit
- val byteCount = this.bufferingJOS.getCount()
- val wholeBytesWritten = directDOS.putBytes(ba, 0, byteCount, finfo)
+ val byteCount = bufferNBits / 8
+ val wholeBytesWritten = {
+ if (bufOS.isFile) {
+ bufOS.close
+ val nBitsPut = try {
+ directDOS.putFile(bufOS.getFile.toPath, bufferNBits.toLong, chunkSizeInBytes, finfo)
+ } finally {
+ // Make sure we delete the file after it was put in the directDOS or
+ // if we encountered an error
+ if (bufOS.isTempFile)
+ bufOS.getFile.delete()
+ }
+ nBitsPut / 8
+ } else
+ directDOS.putBytes(bufOS.getBuf, 0, byteCount.toInt, finfo)
+ }
+
Assert.invariant(byteCount == wholeBytesWritten)
if (nFragBits > 0) {
if (directDOS.isEndOnByteBoundary) {
@@ -802,89 +924,27 @@ class DirectOrBufferedDataOutputStream private[io] (var splitFrom: DirectOrBuffe
origfrag >> (8 - nFragBits)
else
origfrag
- Assert.invariant(directDOS.putLongUnchecked(fragNum, nFragBits, finfo))
- }
- }
- }
- }
-
-}
-
-final class BufferedBlobDataOutputStream private[io] (
- splitFrom: DirectOrBufferedDataOutputStream,
- path: Path,
- lengthInBits: Long,
- blobChunkSizeInBytes: Int,
- isLayer: Boolean = false)
- extends DirectOrBufferedDataOutputStream(splitFrom, isLayer) {
-
- // Data should never be written to a buffer in this BLOB DOS, so there is no
- // need for a bufferingJOS that the parent class would allocate. However, the
- // bufferJOS is used to determine if a DOS is buffering or not. In our case,
- // this is buffering if the JavaOutputStream is the same as bufferingJOS
- // (i.e. null).
- override protected val bufferingJOS = null
-
- // This function is exactly the same as the one it overrides in the parent
- // except that it does not have the assertion that _javaOutputStream is never
- // null. This is because when this BLOB DOS is not direct, the
- // javaOutputStream is set to null (and should never be written to). Once
- // this becomes direct, the _javaOutputStream variable is changed to the
- // direct OutputStream.
- override def getJavaOutputStream() = {
- _javaOutputStream
- }
-
- protected override def deliverContent(directDOS: DirectOrBufferedDataOutputStream, finfo: FormatInfo) = {
- val blobStream =
- try {
- Files.newInputStream(path, StandardOpenOption.READ)
- } catch {
- case e: Exception =>
- throw new BlobIOException(
- "Unable to open BLOB %s for reading: %s".format(
- path.toString,
- e.getMessage()))
- }
-
- val array = new Array[Byte](blobChunkSizeInBytes)
-
- val blobChunkSizeInBits = blobChunkSizeInBytes * 8
-
- var remainingBitsToPut = lengthInBits
- var fileHasData = true
- while (remainingBitsToPut > 0 && fileHasData) {
- val bitsToRead = Math.min(remainingBitsToPut, blobChunkSizeInBits)
- val bytesToRead = (bitsToRead + 7) / 8
- val bytesRead = blobStream.read(array, 0, bytesToRead.toInt)
- if (bytesRead == -1) {
- fileHasData = false
- } else {
- val bitsToPut = Math.min(bytesRead * 8, bitsToRead)
- val ret = directDOS.putByteArray(array, bitsToPut.toInt, finfo)
- if (!ret) {
- blobStream.close()
- throw new BlobIOException(
- "Failed to write BLOB data: %s".format(path.toString))
+ Assert.invariant(directDOS.putLongUnchecked(
+ fragNum, nFragBits, finfo,
+ ignoreByteOrder=this.bufferingJOS.isFile))
}
- remainingBitsToPut -= bitsToPut
}
}
+ bufOS.close()
+ }
- blobStream.close()
+ /**
+ * Clean up any temporary files that were generated
+ */
+ def cleanUp(): Unit = {
+ if (isBuffering && bufferingJOS.isTempFile)
+ bufferingJOS.getFile.delete()
- // calculate the skip bits
- val nFillBits = remainingBitsToPut
- if (nFillBits > 0) {
- val ret = directDOS.skip(nFillBits, finfo)
- if (!ret) {
- throw new BlobIOException(
- "Failed to skip %s bits after BLOB data: %s".format(nFillBits, path.toString))
- }
+ while (_following.isDefined) {
+ _following.get.cleanUp()
}
}
-
}
/**
@@ -906,7 +966,7 @@ class BitOrderChangeException(directDOS: DirectOrBufferedDataOutputStream, finfo
}
}
-class BlobIOException(message: String) extends Exception(message)
+class FileIOException(message: String) extends Exception(message)
object DirectOrBufferedDataOutputStream {
@@ -961,8 +1021,21 @@ object DirectOrBufferedDataOutputStream {
* Factory for creating new ones/
* Passing creator as null indicates no other stream created this one.
*/
- def apply(jos: java.io.OutputStream, creator: DirectOrBufferedDataOutputStream, isLayer: Boolean = false) = {
- val dbdos = new DirectOrBufferedDataOutputStream(creator, isLayer)
+ def apply(
+ jos: java.io.OutputStream,
+ creator: DirectOrBufferedDataOutputStream,
+ isLayer: Boolean = false,
+ chunkSizeInBytes: Int,
+ maxBufferSizeInBytes: Long,
+ tempDirPath: File,
+ maybeExistingFile: Maybe[Path] = Maybe.Nope) = {
+ val dbdos = new DirectOrBufferedDataOutputStream(
+ creator,
+ isLayer,
+ chunkSizeInBytes,
+ maxBufferSizeInBytes,
+ tempDirPath,
+ maybeExistingFile)
dbdos.setJavaOutputStream(jos)
if (creator eq null) {
diff --git a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream.scala b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream.scala
index 014b52a..f8157bf 100644
--- a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream.scala
+++ b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream.scala
@@ -17,9 +17,11 @@
package org.apache.daffodil.io
+import java.io.File
import junit.framework.Assert._
import org.junit.Test
import org.apache.daffodil.schema.annotation.props.gen.BitOrder
+import org.apache.daffodil.util.Maybe
class TestDataOutputStream {
@@ -27,14 +29,14 @@ class TestDataOutputStream {
def newDirectOrBufferedDataOutputStream(jos: java.io.OutputStream, creator: DirectOrBufferedDataOutputStream,
bo: BitOrder = BitOrder.MostSignificantBitFirst) = {
- val os = DirectOrBufferedDataOutputStream(jos, creator)
+ val os = DirectOrBufferedDataOutputStream(jos, creator, false, 4096, 2000 * (1 << 20), new File("."), Maybe.Nope)
os.setPriorBitOrder(bo)
os
}
@Test def testPutLongDirect1_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(-1L, 32, beFinfo)
@@ -52,7 +54,7 @@ class TestDataOutputStream {
@Test def testPutLongDirect1Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(1, 1, beFinfo)
@@ -68,7 +70,7 @@ class TestDataOutputStream {
}
@Test def testPutLongDirect2Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(3, 2, beFinfo)
@@ -84,7 +86,7 @@ class TestDataOutputStream {
}
@Test def testPutLongDirect7Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(0xA5, 7, beFinfo)
@@ -100,7 +102,7 @@ class TestDataOutputStream {
}
@Test def testPutLongDirect8Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(0xA5, 8, beFinfo)
@@ -116,7 +118,7 @@ class TestDataOutputStream {
}
@Test def testPutLongDirect9Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(0xA5A5, 9, beFinfo)
@@ -133,7 +135,7 @@ class TestDataOutputStream {
}
@Test def testPutLongDirect63Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(0xA5A5A5A5A5A5A5A5L, 63, beFinfo)
@@ -156,7 +158,7 @@ class TestDataOutputStream {
}
@Test def testPutLongDirect64Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(0xA5A5A5A5A5A5A5A5L, 64, beFinfo)
@@ -184,7 +186,7 @@ class TestDataOutputStream {
@Test def testPutLongBuffered1_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -207,7 +209,7 @@ class TestDataOutputStream {
@Test def testPutLongBuffered1Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -226,7 +228,7 @@ class TestDataOutputStream {
}
@Test def testPutLongBuffered2Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -245,7 +247,7 @@ class TestDataOutputStream {
}
@Test def testPutLongBuffered7Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -264,7 +266,7 @@ class TestDataOutputStream {
}
@Test def testPutLongBuffered8Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
direct.setPriorBitOrder(BitOrder.MostSignificantBitFirst)
@@ -284,7 +286,7 @@ class TestDataOutputStream {
}
@Test def testPutLongBuffered9Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -304,7 +306,7 @@ class TestDataOutputStream {
}
@Test def testPutLongBuffered63Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -330,7 +332,7 @@ class TestDataOutputStream {
}
@Test def testPutLongBuffered64Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -361,7 +363,7 @@ class TestDataOutputStream {
@Test def testPutLongDirectAndBuffered1_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -389,7 +391,7 @@ class TestDataOutputStream {
@Test def testPutLongDirectAndBuffered1Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -409,7 +411,7 @@ class TestDataOutputStream {
}
@Test def testPutLongDirectAndBuffered2Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -429,7 +431,7 @@ class TestDataOutputStream {
}
@Test def testPutLongDirectAndBuffered7Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -450,7 +452,7 @@ class TestDataOutputStream {
}
@Test def testPutLongDirectAndBuffered8Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -471,7 +473,7 @@ class TestDataOutputStream {
}
@Test def testPutLongDirectAndBuffered9Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -493,7 +495,7 @@ class TestDataOutputStream {
}
@Test def testPutLongDirectAndBuffered63BitPlus1Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -520,7 +522,7 @@ class TestDataOutputStream {
}
@Test def testPutLongDirectAndBuffered63BitPlus63Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -556,7 +558,7 @@ class TestDataOutputStream {
}
@Test def testPutLongDirectAndBuffered64BitPlus64Bit_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -593,7 +595,7 @@ class TestDataOutputStream {
@Test def testPutLong5_4Bits_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(5L, 4, beFinfo)
diff --git a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream2.scala b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream2.scala
index c3cbdca..346236d 100644
--- a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream2.scala
+++ b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream2.scala
@@ -20,6 +20,8 @@ package org.apache.daffodil.io
import junit.framework.Assert._
import org.junit.Test
import java.nio.ByteBuffer
+import java.io.File
+import org.apache.daffodil.util.Maybe
class TestDataOutputStream2 {
@@ -31,8 +33,8 @@ class TestDataOutputStream2 {
@Test def testPutBitBufferDirect0_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
- val out = DirectOrBufferedDataOutputStream(baos, null)
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
+ val out = DirectOrBufferedDataOutputStream(baos, null, false, 4096, 2000 * (1 << 20), new File("."), Maybe.Nope)
val bb = ByteBuffer.wrap(List(0xA5.toByte, 0xBE.toByte, 0xEF.toByte).toArray)
bb.position(1)
@@ -51,8 +53,8 @@ class TestDataOutputStream2 {
@Test def testPutBitBufferDirect1_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
- val out = DirectOrBufferedDataOutputStream(baos, null)
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
+ val out = DirectOrBufferedDataOutputStream(baos, null, false, 4096, 2000 * (1 << 20), new File("."), Maybe.Nope)
val bb = ByteBuffer.wrap(List(0xA5.toByte, 0xBE.toByte, 0xEF.toByte).toArray)
bb.position(1)
@@ -71,8 +73,8 @@ class TestDataOutputStream2 {
@Test def testPutBitBufferDirect7_BE_MSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
- val out = DirectOrBufferedDataOutputStream(baos, null)
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
+ val out = DirectOrBufferedDataOutputStream(baos, null, false, 4096, 2000 * (1 << 20), new File("."), Maybe.Nope)
val bb = ByteBuffer.wrap(List(0xA5.toByte, 0xBE.toByte, 0xEF.toByte).toArray)
bb.position(1)
diff --git a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream3.scala b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream3.scala
index fc7ecc7..1e44dbf 100644
--- a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream3.scala
+++ b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream3.scala
@@ -17,9 +17,11 @@
package org.apache.daffodil.io
+import java.io.File
import junit.framework.Assert._
import org.apache.daffodil.schema.annotation.props.gen.BitOrder
import org.apache.daffodil.schema.annotation.props.gen.ByteOrder
+import org.apache.daffodil.util.Maybe
import org.junit.Test
class TestDataOutputStream3 {
@@ -30,14 +32,14 @@ class TestDataOutputStream3 {
def newDirectOrBufferedDataOutputStream(jos: java.io.OutputStream, creator: DirectOrBufferedDataOutputStream,
bo: BitOrder = BitOrder.MostSignificantBitFirst) = {
- val os = DirectOrBufferedDataOutputStream(jos, creator)
+ val os = DirectOrBufferedDataOutputStream(jos, creator, false, 4096, 2000 * (1 << 20), new File("."), Maybe.Nope)
os.setPriorBitOrder(bo)
os
}
@Test def testPutLongDirect1_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(-1L, 32, lsbfFinfo)
@@ -55,7 +57,7 @@ class TestDataOutputStream3 {
@Test def testPutLongDirect1Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(1, 1, lsbfFinfo)
@@ -71,7 +73,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongDirect2Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(3, 2, lsbfFinfo)
@@ -87,7 +89,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongDirect7Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(0xA5, 7, lsbfFinfo)
@@ -103,7 +105,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongDirect8Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(0xA5, 8, lsbfFinfo)
@@ -119,7 +121,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongDirect9Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(0xA5A5, 9, lsbfFinfo)
@@ -136,7 +138,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongDirect63Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(0xA5A5A5A5A5A5A5A5L, 63, lsbfFinfo)
@@ -159,7 +161,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongDirect64Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(0xA5A5A5A5A5A5A5A5L, 64, lsbfFinfo)
@@ -187,7 +189,7 @@ class TestDataOutputStream3 {
@Test def testPutLongBuffered1_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -209,7 +211,7 @@ class TestDataOutputStream3 {
@Test def testPutLongBuffered1Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -227,7 +229,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongBuffered2Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -245,7 +247,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongBuffered7Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -263,7 +265,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongBuffered8Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -281,7 +283,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongBuffered9Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -300,7 +302,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongBuffered63Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -325,7 +327,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongBuffered64Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -355,7 +357,7 @@ class TestDataOutputStream3 {
@Test def testPutLongDirectAndBuffered1_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -382,7 +384,7 @@ class TestDataOutputStream3 {
@Test def testPutLongDirectAndBuffered1Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null, BitOrder.LeastSignificantBitFirst)
val out = direct.addBuffered
@@ -401,7 +403,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongDirectAndBuffered2Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null, BitOrder.LeastSignificantBitFirst)
val out = direct.addBuffered
@@ -420,7 +422,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongDirectAndBuffered7Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
direct.setPriorBitOrder(BitOrder.LeastSignificantBitFirst)
val out = direct.addBuffered
@@ -441,7 +443,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongDirectAndBuffered8Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -461,7 +463,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongDirectAndBuffered9Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null, BitOrder.LeastSignificantBitFirst)
val out = direct.addBuffered
@@ -482,7 +484,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongDirectAndBuffered63BitPlus1Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null, BitOrder.LeastSignificantBitFirst)
val out = direct.addBuffered
@@ -508,7 +510,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongDirectAndBuffered63BitPlus63Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null, BitOrder.LeastSignificantBitFirst)
val out = direct.addBuffered
@@ -543,7 +545,7 @@ class TestDataOutputStream3 {
}
@Test def testPutLongDirectAndBuffered64BitPlus64Bit_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val direct = newDirectOrBufferedDataOutputStream(baos, null)
val out = direct.addBuffered
@@ -579,7 +581,7 @@ class TestDataOutputStream3 {
@Test def testPutLong5_4Bits_LE_LSBF {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val out = newDirectOrBufferedDataOutputStream(baos, null)
out.putLong(5L, 4, lsbfFinfo)
diff --git a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream4.scala b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream4.scala
index 48ae28f..f30cd9c 100644
--- a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream4.scala
+++ b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDataOutputStream4.scala
@@ -17,10 +17,12 @@
package org.apache.daffodil.io
+import java.io.File
import junit.framework.Assert._
import org.junit.Test
import passera.unsigned.ULong
import org.apache.daffodil.schema.annotation.props.gen.BitOrder
+import org.apache.daffodil.util.Maybe
class TestDataOutputStream4 {
@@ -28,8 +30,8 @@ class TestDataOutputStream4 {
finfo.fillByte = 0.toByte
def setup(setAbs: Boolean = true, bitOrd: BitOrder = BitOrder.MostSignificantBitFirst) = {
- val baos = new ByteArrayOutputStreamWithGetBuf()
- val direct = DirectOrBufferedDataOutputStream(baos, null)
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
+ val direct = DirectOrBufferedDataOutputStream(baos, null, false, 4096, 2000 * (1 << 20), new File("."), Maybe.Nope)
direct.setPriorBitOrder(bitOrd)
direct.putLong(0x5a5a5, 19, finfo)
@@ -61,7 +63,7 @@ class TestDataOutputStream4 {
(baos, direct, out, out2)
}
- private def checkResults(baos: ByteArrayOutputStreamWithGetBuf): Unit = {
+ private def checkResults(baos: ByteArrayOrFileOutputStream): Unit = {
val buf = baos.getBuf()
assertEquals(0xB4.toByte, buf(0))
diff --git a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDirectOrBufferedDataOutputStream.scala b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDirectOrBufferedDataOutputStream.scala
index 867f9a6..94b5628 100644
--- a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDirectOrBufferedDataOutputStream.scala
+++ b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDirectOrBufferedDataOutputStream.scala
@@ -22,12 +22,14 @@ import junit.framework.Assert._
import java.io.ByteArrayInputStream
import java.io.InputStreamReader
+import java.io.File
import org.apache.commons.io.IOUtils
import org.apache.daffodil.schema.annotation.props.gen.BitOrder
+import org.apache.daffodil.util.Maybe
class TestDirectOrBufferedDataOutputStream {
- private def getString(baos: ByteArrayOutputStreamWithGetBuf) = {
+ private def getString(baos: ByteArrayOrFileOutputStream) = {
val is = new ByteArrayInputStream(baos.getBuf)
val ir = new InputStreamReader(is, "ascii")
val line = IOUtils.toString(ir)
@@ -37,7 +39,7 @@ class TestDirectOrBufferedDataOutputStream {
def newDirectOrBufferedDataOutputStream(jos: java.io.OutputStream, creator: DirectOrBufferedDataOutputStream,
bo: BitOrder = BitOrder.MostSignificantBitFirst) = {
- val os = DirectOrBufferedDataOutputStream(jos, creator)
+ val os = DirectOrBufferedDataOutputStream(jos, creator, false, 4096, 2000 * (1 << 20), new File("."), Maybe.Nope)
os.setPriorBitOrder(bo)
os
}
@@ -46,7 +48,7 @@ class TestDirectOrBufferedDataOutputStream {
* if that happens.
*/
@Test def testToStringDoesNotThrow {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val layered = newDirectOrBufferedDataOutputStream(baos, null)
assertFalse(layered.toString().isEmpty())
}
@@ -55,7 +57,7 @@ class TestDirectOrBufferedDataOutputStream {
@Test def testCollapsingBufferIntoDirect1 {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val layered = newDirectOrBufferedDataOutputStream(baos, null)
val hw = "Hello World!"
@@ -82,7 +84,7 @@ class TestDirectOrBufferedDataOutputStream {
@Test def testCollapsingFinishedBufferIntoLayered {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val layered = newDirectOrBufferedDataOutputStream(baos, null)
val hw = "Hello World!"
@@ -111,7 +113,7 @@ class TestDirectOrBufferedDataOutputStream {
@Test def testCollapsingTwoBuffersIntoDirect {
- val baos = new ByteArrayOutputStreamWithGetBuf()
+ val baos = new ByteArrayOrFileOutputStream(2000 * (1 << 20), new File("."), Maybe.Nope)
val layered = newDirectOrBufferedDataOutputStream(baos, null)
val hw = "Hello World!"
diff --git a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd
index 6923c7c..77c20aa 100644
--- a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd
+++ b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd
@@ -119,6 +119,32 @@
</xs:documentation>
</xs:annotation>
</xs:element>
+ <xs:element name="outputStreamChunkSizeInBytes" type="xs:int" default="65536" minOccurs="0">
+ <xs:annotation>
+ <xs:documentation>
+ When writing file data to the output stream during unparse, this
+ is the maximum number of bytes to write at a time.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:element>
+ <xs:element name="maxByteArrayOutputStreamBufferSizeInBytes" type="xs:long" default="2097152000" minOccurs="0">
+ <xs:annotation>
+ <xs:documentation>
+ When unparsing, this is the maximum size of the buffer that the
+ ByteArrayOutputStream can grow to before switching to a file based
+ output stream.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:element>
+ <xs:element name="tempFilePath" type="xs:string" default="" minOccurs="0">
+ <xs:annotation>
+ <xs:documentation>
+ When unparsing, use this path to store temporary files that may be genrated.
+ The default value (empty string) will result in the use of the java.io.tmpdir
+ property being used as the path.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:element>
<xs:element name="defaultInitialRegexMatchLimitInChars" type="xs:int" default="32" minOccurs="0">
<xs:annotation>
<xs:documentation>
diff --git a/daffodil-propgen/src/main/scala/org/apache/daffodil/propGen/TunableGenerator.scala b/daffodil-propgen/src/main/scala/org/apache/daffodil/propGen/TunableGenerator.scala
index d62e280..d5b94b8 100644
--- a/daffodil-propgen/src/main/scala/org/apache/daffodil/propGen/TunableGenerator.scala
+++ b/daffodil-propgen/src/main/scala/org/apache/daffodil/propGen/TunableGenerator.scala
@@ -147,6 +147,9 @@ class TunableGenerator(schemaRootConfig: scala.xml.Node, schemaRootExt: scala.xm
} else if (!schemaType.startsWith("xs:")) {
// non-primitive type, assume a single enum
new EnumTunable(schemaName, schemaType, schemaDefault)
+ } else if (schemaName == "tempFilePath") {
+ // special case, creates actual file object instead of string
+ new TempFilePathTunable()
} else {
// primitive type
new PrimitiveTunable(schemaName, schemaType, schemaDefault)
@@ -201,6 +204,11 @@ class PrimitiveTunable(name: String, schemaType: String, schemaDefault: String)
override val scalaConversion = s"""case "${name}" => this.copy(${name} = value.to${scalaType})"""
}
+class TempFilePathTunable() extends TunableBase {
+ override val scalaDefinition = s"""val tempFilePath: java.io.File = new java.io.File(System.getProperty(\"java.io.tmpdir\"))"""
+ override val scalaConversion = s"""case "tempFilePath" => this.copy(tempFilePath = new java.io.File(value))"""
+}
+
class EnumTunable(name: String, schemaType: String, schemaDefault: String)
extends TunableBase {
diff --git a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/BlobLengthUnparser.scala b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/BlobLengthUnparser.scala
index 3bdbad0..e5c0238 100644
--- a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/BlobLengthUnparser.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/BlobLengthUnparser.scala
@@ -87,8 +87,8 @@ abstract class BlobUnparserBase(override val context: ElementRuntimeData)
// second data output stream will then be delivered, finally making it the
// direct stream.
val dos = state.dataOutputStream
- val newDOS = dos.addBufferedBlob(path, lengthInBits, state.tunable.blobChunkSizeInBytes, state)
- state.dataOutputStream = newDOS
+ val newStream = dos.addBufferedBlob(path, lengthInBits, state.tunable.blobChunkSizeInBytes, state)
+ state.dataOutputStream = newStream
dos.setFinished(state)
}
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala
index 9c9e950..a218dc2 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala
@@ -154,7 +154,13 @@ abstract class LayerTransformer()
val jos = wrapJavaOutputStream(s, state)
val limitedJOS = wrapLimitingStream(jos, state)
val encodedOutputStream = wrapLayerEncoder(limitedJOS)
- val newDOS = DirectOrBufferedDataOutputStream(encodedOutputStream, null, isLayer = true)
+ val newDOS = DirectOrBufferedDataOutputStream(
+ encodedOutputStream,
+ null,
+ isLayer = true,
+ s.chunkSizeInBytes,
+ s.maxBufferSizeInBytes,
+ s.tempDirPath)
newDOS.setPriorBitOrder(BitOrder.MostSignificantBitFirst)
newDOS.setAbsStartingBitPos0b(ULong(0L))
newDOS.setDebugging(s.areDebugging)
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DataProcessor.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DataProcessor.scala
index 759baff..9682e61 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DataProcessor.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/DataProcessor.scala
@@ -22,6 +22,7 @@ import java.io.ObjectOutputStream
import java.nio.channels.Channels
import java.nio.CharBuffer
import java.nio.file.Files
+import java.nio.file.Paths
import java.nio.LongBuffer
import java.util.zip.GZIPOutputStream
@@ -55,7 +56,7 @@ import org.apache.daffodil.io.DirectOrBufferedDataOutputStream
import org.apache.daffodil.io.InputSourceDataInputStream
import org.apache.daffodil.util.LogLevel
import org.apache.daffodil.io.BitOrderChangeException
-import org.apache.daffodil.io.BlobIOException
+import org.apache.daffodil.io.FileIOException
import org.apache.daffodil.infoset._
import org.apache.daffodil.processors.parsers.ParseError
import org.apache.daffodil.processors.parsers.Parser
@@ -304,7 +305,14 @@ class DataProcessor(val ssrd: SchemaSetRuntimeData)
}
def unparse(inputter: InfosetInputter, outStream: java.io.OutputStream) = {
- val out = DirectOrBufferedDataOutputStream(outStream, null) // null means no other stream created this one.
+ val out = DirectOrBufferedDataOutputStream(
+ outStream,
+ null, // null means no other stream created this one.
+ isLayer=false,
+ tunablesObj.outputStreamChunkSizeInBytes,
+ tunablesObj.maxByteArrayOutputStreamBufferSizeInBytes,
+ tunablesObj.tempFilePath)
+
inputter.initialize(ssrd.elementRuntimeData, getTunables())
val unparserState =
UState.createInitialUState(
@@ -364,6 +372,8 @@ class DataProcessor(val ssrd: SchemaSetRuntimeData)
unparserState.unparseResult
}
case th: Throwable => throw th
+ } finally {
+ unparserState.dataOutputStream.cleanUp
}
res
}
@@ -418,8 +428,8 @@ class DataProcessor(val ssrd: SchemaSetRuntimeData)
} catch {
case boc: BitOrderChangeException =>
state.SDE(boc)
- case bio: BlobIOException =>
- state.SDE(bio)
+ case fio: FileIOException =>
+ state.SDE(fio)
}
log(LogLevel.Debug, "%s final stream for %s finished.", this, state)
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/unparsers/UState.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/unparsers/UState.scala
index d908659..a2462fa 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/unparsers/UState.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/unparsers/UState.scala
@@ -20,6 +20,7 @@ package org.apache.daffodil.processors.unparsers
import java.io.ByteArrayOutputStream
import java.nio.CharBuffer
import java.nio.LongBuffer
+import java.nio.file.Paths
import scala.Left
import scala.collection.mutable
@@ -496,7 +497,13 @@ final class UStateMain private (
override lazy val withByteArrayOutputStream = new LocalStack[(ByteArrayOutputStream, DirectOrBufferedDataOutputStream)](
{
val baos = new ByteArrayOutputStream() // TODO: PERFORMANCE: Allocates new object. Can reuse one from an onStack/pool via reset()
- val dos = DirectOrBufferedDataOutputStream(baos, null)
+ val dos = DirectOrBufferedDataOutputStream(
+ baos,
+ null,
+ false,
+ tunable.outputStreamChunkSizeInBytes,
+ tunable.maxByteArrayOutputStreamBufferSizeInBytes,
+ tunable.tempFilePath)
(baos, dos)
},
pair => pair match {
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/testUnparserFileBuffering.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/testUnparserFileBuffering.tdml
new file mode 100644
index 0000000..bd1c5af
--- /dev/null
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/testUnparserFileBuffering.tdml
@@ -0,0 +1,798 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<tdml:testSuite xmlns:tdml="http://www.ibm.com/xmlns/dfdl/testData"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:ex="http://example.com"
+ xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext" suiteName="unparserFileBuffering">
+
+ <tdml:defineConfig name="cfg_forceFileBuffering">
+ <daf:tunables xmlns="http://www.w3.org/2001/XMLSchema"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema">
+ <daf:maxByteArrayOutputStreamBufferSizeInBytes>0</daf:maxByteArrayOutputStreamBufferSizeInBytes>
+ </daf:tunables>
+ </tdml:defineConfig>
+
+ <tdml:defineSchema name="fixedLengthStrings">
+ <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd" />
+ <dfdl:format ref="ex:GeneralFormat" textPadKind='padChar' />
+ <xs:element name="e1" dfdl:lengthKind="explicit" dfdl:length="5" type="xs:string" dfdl:truncateSpecifiedLengthString="yes"/>
+ <xs:element name="e2" dfdl:lengthKind="explicit" dfdl:length="1" type="xs:string" />
+ <xs:element name="e3" dfdl:lengthKind="explicit" dfdl:length="5" type="xs:string" dfdl:textPadKind='none'/>
+
+ <xs:element name="e4" dfdl:lengthKind="implicit">
+ <xs:complexType>
+ <xs:sequence dfdl:initiator="[" dfdl:separator="," dfdl:terminator="]">
+ <xs:element name="s1" dfdl:lengthKind="explicit" dfdl:length="0" type="xs:string" dfdl:textPadKind='none'/>
+ <xs:element name="s2" dfdl:lengthKind="explicit" dfdl:length="1" type="xs:string" dfdl:textPadKind='none' default="broken"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ </tdml:defineSchema>
+
+ <tdml:defineSchema name="illegalChars">
+ <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd" />
+ <dfdl:format ref="ex:GeneralFormat"/>
+
+ <xs:element name="e1" dfdl:lengthKind="delimited" type="xs:string"/>
+
+ </tdml:defineSchema>
+
+<!--
+ Test Name: apostrophe_01
+ Schema: fixedLengthStrings
+ Purpose: This test demonstrates that Daffodil can handle unparsing an apostrophe
+ in the infoset, and parsing an apostrophe in the document.
+-->
+
+ <tdml:unparserTestCase name="apostrophe_01" root="e2" model="fixedLengthStrings" roundTrip="true"
+ config="cfg_forceFileBuffering">
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e2>'</ex:e2>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ <tdml:document>'</tdml:document>
+
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: puaInfosetChars_01
+ Schema: illegalChars
+ Purpose: When parsing illegal XML characters, they get mapped to PUA. This test demonstrates that when unparsing these mapped infosets, the characters get mapped back to their original values.
+-->
+
+ <tdml:parserTestCase name="puaInfosetChars_01" root="e1" model="illegalChars" roundTrip="true"
+ config="cfg_forceFileBuffering">
+ <tdml:document>
+ <tdml:documentPart type="byte">00</tdml:documentPart>
+ </tdml:document>
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e1></ex:e1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ </tdml:parserTestCase>
+
+<!--
+ Test Name: puaInfosetChars_02
+ Schema: illegalChars
+ Purpose: When parsing illegal XML characters, they get mapped to PUA. This test demonstrates that when unparsing these mapped infosets, the characters get mapped back to their original values.
+-->
+
+ <tdml:parserTestCase name="puaInfosetChars_02" root="e1" model="illegalChars" roundTrip="true"
+ config="cfg_forceFileBuffering">
+ <tdml:document>
+ <tdml:documentPart type="byte">010203</tdml:documentPart>
+ </tdml:document>
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e1></ex:e1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ </tdml:parserTestCase>
+
+<!--
+ Test Name: puaInfosetChars_03
+ Schema: illegalChars
+ Purpose: When parsing illegal XML characters, they get mapped to PUA. This test demonstrates that unparsing and unparsing actual PUA values is handled correctly.
+-->
+
+ <tdml:parserTestCase name="puaInfosetChars_03" root="e1" model="illegalChars" roundTrip="true"
+ config="cfg_forceFileBuffering">
+ <tdml:document></tdml:document>
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e1></ex:e1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ </tdml:parserTestCase>
+
+<!--
+ Test Name: puaInfosetChars_04
+ Schema: illegalChars
+ Purpose: When parsing illegal XML characters, they get mapped to PUA. This test demonstrates that when unparsing these mapped infosets, the characters get mapped back to their original values.
+-->
+
+ <tdml:unparserTestCase name="puaInfosetChars_04" root="e1" model="illegalChars" roundTrip="true"
+ config="cfg_forceFileBuffering">
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e1></ex:e1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ <tdml:document></tdml:document>
+
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: unparseFixedLengthString01
+ Schema: fixedLengthStrings
+ Purpose: This is a basic test of the unparse capabilities of Daffodil and the TDML Runner
+-->
+
+ <tdml:unparserTestCase name="unparseFixedLengthString01" root="e1" model="fixedLengthStrings" roundTrip="true"
+ config="cfg_forceFileBuffering">
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e1>Hello</ex:e1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ <tdml:document>Hello</tdml:document>
+
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: parseFixedLengthString01
+ Schema: fixedLengthStrings
+ Purpose: This is a basic test of the roundtrip capabilities of Daffodil and the TDML Runner for
+ a parserTestCase.
+-->
+
+ <tdml:parserTestCase name="parseFixedLengthString01" root="e1" model="fixedLengthStrings" roundTrip="true"
+ config="cfg_forceFileBuffering">
+ <tdml:document>Hello</tdml:document>
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e1>Hello</ex:e1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ </tdml:parserTestCase>
+
+<!--
+ Test Name: parseFixedLengthStringLength0
+ Schema: fixedLengthStrings
+ Purpose: Test to see what happens when the fixed length of a string is '0' and when a default value is supplied
+-->
+
+ <tdml:parserTestCase name="parseFixedLengthStringLength0" root="e4" model="fixedLengthStrings" roundTrip="true"
+ config="cfg_forceFileBuffering">
+ <tdml:document>[,2]</tdml:document>
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e4 xmlns:ex="http://example.com">
+ <ex:s1></ex:s1>
+ <ex:s2>2</ex:s2>
+ </ex:e4>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ </tdml:parserTestCase>
+
+
+<!--
+ Test Name: unparseFixedLengthString02
+ Schema: fixedLengthStrings
+ Purpose: This is a basic test of the unparse capabilities of Daffodil and the TDML Runner
+-->
+
+ <tdml:unparserTestCase name="unparseFixedLengthString02" root="e2" model="fixedLengthStrings" roundTrip="true"
+ config="cfg_forceFileBuffering">
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e2>0</ex:e2>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ <tdml:document>0</tdml:document>
+
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: unparseFixedLengthString03
+ Schema: fixedLengthStrings
+ Purpose: This is a basic test of the unparse capabilities of Daffodil and the TDML Runner
+ In this case the test is referencing an external schema rather than an embedded one.
+-->
+
+ <tdml:unparserTestCase name="unparseFixedLengthString03" root="e1" model="generalSchema.dfdl.xsd"
+ config="cfg_forceFileBuffering">
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e1>Hello</ex:e1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ <tdml:document>Hello</tdml:document>
+
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: negativeUnparseTest01
+ Schema: fixedLengthStrings
+ Purpose: This is a test of how the TDML Runner handles a failed Unparse test.
+ In this case, the expected output does not match the unparsed output.
+-->
+
+ <tdml:unparserTestCase name="negativeUnparseTest01" root="e2" model="fixedLengthStrings"
+ config="cfg_forceFileBuffering">
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e2>0</ex:e2>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ <tdml:document>incorrect</tdml:document>
+
+ <tdml:errors>
+ <tdml:error>TDMLException</tdml:error>
+ <tdml:error>output data length</tdml:error>
+ <tdml:error>length 1</tdml:error>
+ <tdml:error>'0'</tdml:error>
+ <tdml:error>length 9</tdml:error>
+ <tdml:error>'incorrect'</tdml:error>
+ </tdml:errors>
+
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: negativeUnparseTest02
+ Schema: fixedLengthStrings
+ Purpose: This is a test of how the TDML Runner handles a failed Unparse test.
+ In this case, the expected output does not match the unparsed output.
+-->
+
+ <tdml:unparserTestCase name="negativeUnparseTest02" root="e2" model="fixedLengthStrings"
+ config="cfg_forceFileBuffering">
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e2>X</ex:e2>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ <tdml:document>Q</tdml:document>
+
+ <tdml:errors>
+ <tdml:error>TDMLException</tdml:error>
+ <tdml:error>data differs</tdml:error>
+ <tdml:error>Expected 'Q'</tdml:error>
+ <tdml:error>Actual was 'X'</tdml:error>
+ </tdml:errors>
+
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: negativeUnparseTest03
+ Schema: fixedLengthStrings
+ Purpose: This is a test of how the TDML Runner handles a failed Unparse test.
+ In this case, the input infoset is more data than the element should allow, so
+ an unparse error should occur.
+-->
+
+ <tdml:unparserTestCase name="negativeUnparseTest03" root="e1" model="fixedLengthStrings"
+ config="cfg_forceFileBuffering">
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e1>too much data</ex:e1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ <tdml:document>too much data</tdml:document>
+
+ <tdml:errors>
+ <tdml:error>output data length</tdml:error>
+ <tdml:error>length 5</tdml:error>
+ <tdml:error>doesn't match expected</tdml:error>
+ <tdml:error>length 13</tdml:error>
+ <tdml:error>too much data</tdml:error>
+ </tdml:errors>
+
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: negativeUnparseTest04
+ Schema: fixedLengthStrings
+ Purpose: This is a test of how the TDML Runner handles a failed Unparse test.
+ In this case, the input infoset is less data than the element should allow, so
+ an unparse error should occur.
+-->
+
+ <tdml:unparserTestCase name="negativeUnparseTest04" root="e3" model="fixedLengthStrings"
+ config="cfg_forceFileBuffering">
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e3>1</ex:e3>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ <tdml:document>1</tdml:document>
+
+ <tdml:errors>
+ <tdml:error>output data length 5</tdml:error>
+ <tdml:error>length 1</tdml:error>
+ <tdml:error>doesn't match</tdml:error>
+ </tdml:errors>
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: negativeUnparseTest05
+ Schema: fixedLengthStrings
+ Purpose: This is a test of how the TDML Runner handles a failed Unparse test.
+ In this case, the expected output does not match the unparsed output.
+-->
+
+ <tdml:unparserTestCase name="negativeUnparseTest05" root="e2" model="fixedLengthStrings"
+ config="cfg_forceFileBuffering">
+
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e2>X</ex:e2>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ <tdml:document>Xtra</tdml:document>
+
+ <tdml:errors>
+ <tdml:error>TDMLException</tdml:error>
+ <tdml:error>output data length 1</tdml:error>
+ <tdml:error>'X'</tdml:error>
+ <tdml:error>doesn't match expected length 4</tdml:error>
+ <tdml:error>'Xtra'</tdml:error>
+ </tdml:errors>
+
+ </tdml:unparserTestCase>
+
+ <tdml:defineSchema name="delimitedStrings">
+ <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd" />
+
+ <dfdl:format ref="ex:GeneralFormat" encoding="ascii"
+ lengthUnits="bytes" outputNewLine="%CR;%LF;"/>
+
+ <xs:element name="e1" dfdl:lengthKind="implicit">
+ <xs:complexType>
+ <xs:sequence dfdl:initiator="[" dfdl:separator="," dfdl:terminator="]">
+ <xs:element name="s1" type="xs:string" dfdl:lengthKind="delimited"/>
+ <xs:element name="s2" type="xs:string" dfdl:lengthKind="delimited"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="e2" dfdl:lengthKind="delimited">
+ <xs:complexType>
+ <xs:sequence dfdl:separator=",">
+ <xs:element name="s1" type="xs:string" dfdl:lengthKind="delimited" dfdl:initiator="1."/>
+ <xs:element name="s2" type="xs:string" dfdl:lengthKind="delimited" dfdl:initiator="2."/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="e3" dfdl:lengthKind="delimited">
+ <xs:complexType>
+ <xs:sequence dfdl:separator="">
+ <xs:element name="s1" type="xs:string" dfdl:lengthKind="delimited" dfdl:terminator="!"/>
+ <xs:element name="s2" type="xs:string" dfdl:lengthKind="delimited" dfdl:terminator="!"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="e4" dfdl:lengthKind="delimited">
+ <xs:complexType>
+ <xs:sequence dfdl:separator="">
+ <xs:element name="s1" type="xs:string" dfdl:lengthKind="delimited" dfdl:terminator="!"/>
+ <xs:element name="s2" type="xs:string" dfdl:lengthKind="delimited" dfdl:terminator="?"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="e5" type="xs:string" dfdl:lengthKind="delimited" dfdl:initiator="!"/>
+
+ </tdml:defineSchema>
+
+<!--
+ Test Name: unparseDelimitedString01
+ Schema: delimitedStrings
+ Purpose: This test demonstrates unparsing a sequence of delimited strings
+-->
+ <tdml:unparserTestCase name="unparseDelimitedString01" model="delimitedStrings" root="e1" roundTrip="true"
+ config="cfg_forceFileBuffering">
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e1 xmlns:ex="http://example.com">
+ <ex:s1>1</ex:s1>
+ <ex:s2>2</ex:s2>
+ </ex:e1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ <tdml:document>[1,2]</tdml:document>
+ </tdml:unparserTestCase>
+
+ <tdml:parserTestCase name="parseDelimitedString01" model="delimitedStrings" root="e1"
+ config="cfg_forceFileBuffering">
+ <tdml:document>[1,2]</tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e1 xmlns:ex="http://example.com">
+ <ex:s1>1</ex:s1>
+ <ex:s2>2</ex:s2>
+ </ex:e1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+
+ </tdml:parserTestCase>
+
+<!--
+ Test Name: unparseDelimitedString02
+ Schema: delimitedStrings
+ Purpose: This test demonstrates unparsing a sequence of delimited strings. In this case
+ there is too much data and an error should occur
+-->
+ <tdml:unparserTestCase name="unparseDelimitedString02" model="delimitedStrings" root="e1" roundTrip="false"
+ config="cfg_forceFileBuffering">
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e1 xmlns:ex="http://example.com">
+ <ex:s1>1</ex:s1>
+ <ex:s2>2</ex:s2>
+ <ex:s2>2</ex:s2>
+ <ex:s2>2</ex:s2>
+ </ex:e1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ <tdml:errors>
+ <tdml:error>Unparse Error</tdml:error>
+ <tdml:error>{http://example.com}s2</tdml:error>
+ <tdml:error>expected element end</tdml:error>
+ <tdml:error>{http://example.com}e1</tdml:error>
+ </tdml:errors>
+
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: unparseDelimitedString03
+ Schema: delimitedStrings
+ Purpose: This test demonstrates unparsing a sequence of delimited strings. In this case
+ there is not enough data and an error should occur
+-->
+ <tdml:unparserTestCase name="unparseDelimitedString03" model="delimitedStrings" root="e1" roundTrip="false"
+ config="cfg_forceFileBuffering">
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e1 xmlns:ex="http://example.com">
+ <ex:s1>1</ex:s1>
+ </ex:e1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ <tdml:errors>
+ <tdml:error>Unparse Error</tdml:error>
+ <tdml:error>Expected element start event</tdml:error>
+ <tdml:error>s2</tdml:error>
+ <tdml:error>received element end event</tdml:error>
+ <tdml:error>e1</tdml:error>
+ </tdml:errors>
+
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: unparseDelimitedString04
+ Schema: delimitedStrings
+ Purpose: This test demonstrates unparsing a sequence of delimited strings. In this case the data is delimited by initiators.
+-->
+
+ <tdml:unparserTestCase name="unparseDelimitedString04" model="delimitedStrings" root="e2" roundTrip="true"
+ config="cfg_forceFileBuffering">
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e2 xmlns:ex="http://example.com">
+ <ex:s1>1</ex:s1>
+ <ex:s2>2</ex:s2>
+ </ex:e2>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ <tdml:document>1.1,2.2</tdml:document>
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: unparseDelimitedString05
+ Schema: delimitedStrings
+ Purpose: This test demonstrates unparsing a sequence of delimited strings. In this case the data is delimited by terminators.
+-->
+ <tdml:unparserTestCase name="unparseDelimitedString05" model="delimitedStrings" root="e3" roundTrip="true"
+ config="cfg_forceFileBuffering">
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e3 xmlns:ex="http://example.com">
+ <ex:s1>1</ex:s1>
+ <ex:s2>2</ex:s2>
+ </ex:e3>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ <tdml:document>1!2!</tdml:document>
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: unparseDelimitedString06
+ Schema: delimitedStrings
+ Purpose: This test demonstrates unparsing a sequence of delimited strings. In this case the data is delimited by terminators.
+-->
+ <tdml:unparserTestCase name="unparseDelimitedString06" model="delimitedStrings" root="e4" roundTrip="true"
+ config="cfg_forceFileBuffering">
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e4 xmlns:ex="http://example.com">
+ <ex:s1>1</ex:s1>
+ <ex:s2>2</ex:s2>
+ </ex:e4>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ <tdml:document>1!2?</tdml:document>
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: unparseDelimitedString07
+ Schema: delimitedStrings
+ Purpose: This test demonstrates unparsing a delimited string. In this case the data is delimited by initiators.
+-->
+
+ <tdml:unparserTestCase name="unparseDelimitedString07" model="delimitedStrings" root="e5" roundTrip="true"
+ config="cfg_forceFileBuffering">
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e5>exclaim</ex:e5>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ <tdml:document>!exclaim</tdml:document>
+ </tdml:unparserTestCase>
+
+
+ <tdml:defineSchema name="alignmentPaddingOVC">
+ <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd" />
+
+ <dfdl:format ref="ex:GeneralFormat" encoding="ascii"
+ lengthUnits="bytes" outputNewLine="%CR;%LF;" alignment="1" alignmentUnits="bytes"
+ textNumberPadCharacter="x" textPadKind="padChar" textTrimKind="padChar"
+ fillByte="%NUL;"/>
+
+ <xs:element name="e1">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="length" type="xs:int" dfdl:textNumberJustification="right" dfdl:lengthKind="explicit" dfdl:textOutputMinLength="4" dfdl:length="4" dfdl:outputValueCalc="{ dfdl:valueLength(../ex:payload, 'bits') }" />
+ <xs:element name="payload">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="sublength" type="xs:unsignedInt" dfdl:representation="binary"
+ dfdl:alignment="1" dfdl:alignmentUnits="bits"
+ dfdl:lengthKind="explicit" dfdl:length="4" dfdl:lengthUnits="bits"/>
+ <xs:element name="subfield1" type="xs:unsignedInt" dfdl:representation="binary"
+ dfdl:alignment="1" dfdl:alignmentUnits="bits"
+ dfdl:lengthKind="explicit" dfdl:length="{ ../ex:sublength }" dfdl:lengthUnits="bits" />
+ <xs:element name="subfield2" type="xs:unsignedInt" dfdl:representation="binary"
+ dfdl:alignment="8" dfdl:alignmentUnits="bits"
+ dfdl:lengthKind="explicit" dfdl:length="4" dfdl:lengthUnits="bits" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="e2">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="length" type="xs:int" dfdl:textNumberJustification="left" dfdl:lengthKind="explicit" dfdl:textOutputMinLength="4" dfdl:length="4" dfdl:outputValueCalc="{ dfdl:valueLength(../ex:payload, 'bits') }" />
+ <xs:element name="payload">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="sublength" type="xs:unsignedInt" dfdl:representation="binary"
+ dfdl:alignment="1" dfdl:alignmentUnits="bits"
+ dfdl:lengthKind="explicit" dfdl:length="4" dfdl:lengthUnits="bits" />
+ <xs:element name="subfield1" type="xs:unsignedInt" dfdl:representation="binary"
+ dfdl:alignment="1" dfdl:alignmentUnits="bits"
+ dfdl:lengthKind="explicit" dfdl:length="{ ../ex:sublength }" dfdl:lengthUnits="bits" />
+ <xs:element name="subfield2" type="xs:unsignedInt" dfdl:representation="binary"
+ dfdl:alignment="8" dfdl:alignmentUnits="bits"
+ dfdl:lengthKind="explicit" dfdl:length="4" dfdl:lengthUnits="bits" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="e3">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="length" type="xs:int" dfdl:textNumberJustification="center" dfdl:lengthKind="explicit" dfdl:textOutputMinLength="4" dfdl:length="4" dfdl:outputValueCalc="{ dfdl:valueLength(../ex:payload, 'bits') }" />
+ <xs:element name="payload">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="sublength" type="xs:unsignedInt" dfdl:representation="binary"
+ dfdl:alignment="1" dfdl:alignmentUnits="bits"
+ dfdl:lengthKind="explicit" dfdl:length="4" dfdl:lengthUnits="bits" />
+ <xs:element name="subfield1" type="xs:unsignedInt" dfdl:representation="binary"
+ dfdl:alignment="1" dfdl:alignmentUnits="bits"
+ dfdl:lengthKind="explicit" dfdl:length="{ ../ex:sublength }" dfdl:lengthUnits="bits" />
+ <xs:element name="subfield2" type="xs:unsignedInt" dfdl:representation="binary"
+ dfdl:alignment="8" dfdl:alignmentUnits="bits"
+ dfdl:lengthKind="explicit" dfdl:length="4" dfdl:lengthUnits="bits" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="e4">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="length" type="xs:int"
+ dfdl:representation="text"
+ dfdl:textPadKind="padChar"
+ dfdl:textTrimKind="padChar"
+ dfdl:textNumberPadCharacter="0"
+ dfdl:textNumberJustification="right"
+ dfdl:lengthKind="explicit"
+ dfdl:textOutputMinLength="4"
+ dfdl:length="1"
+ dfdl:outputValueCalc="{ dfdl:valueLength(../ex:payload, 'bytes') }" />
+ <xs:element name="payload" type="xs:string" dfdl:lengthKind="delimited" dfdl:terminator=";"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ </tdml:defineSchema>
+
+ <!--
+ Test Name: alignmentPaddingOVC1
+ Schema: delimitedStrings
+ Purpose: This test demonstrates roundtrip of an element with padding, OVC, and alignment
+ -->
+ <tdml:unparserTestCase name="alignmentPaddingOVC1" model="alignmentPaddingOVC" root="e1"
+ config="cfg_forceFileBuffering">
+ <tdml:document>
+ <tdml:documentPart type="text">xx20</tdml:documentPart><!-- length of below fields in bits -->
+ <tdml:documentPart type="bits">0101</tdml:documentPart><!-- subfield1 length -->
+ <tdml:documentPart type="bits">10101</tdml:documentPart><!-- subfield1 -->
+ <tdml:documentPart type="bits">0000000</tdml:documentPart><!-- alignment skip -->
+ <tdml:documentPart type="bits">1010</tdml:documentPart><!-- subfield1 -->
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e1 xmlns:ex="http://example.com">
+ <ex:length>20</ex:length>
+ <ex:payload>
+ <ex:sublength>5</ex:sublength>
+ <ex:subfield1>21</ex:subfield1>
+ <ex:subfield2>10</ex:subfield2>
+ </ex:payload>
+ </ex:e1>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: alignmentPaddingOVC2
+ Schema: delimitedStrings
+ Purpose: This test demonstrates roundtrip of an element with padding, OVC, and alignment
+-->
+ <tdml:unparserTestCase name="alignmentPaddingOVC2" model="alignmentPaddingOVC" root="e2"
+ config="cfg_forceFileBuffering">
+ <tdml:document>
+ <tdml:documentPart type="text">20xx</tdml:documentPart><!-- length of below fields in bits -->
+ <tdml:documentPart type="bits">0101</tdml:documentPart><!-- subfield1 length -->
+ <tdml:documentPart type="bits">10101</tdml:documentPart><!-- subfield1 -->
+ <tdml:documentPart type="bits">0000000</tdml:documentPart><!-- alignment skip -->
+ <tdml:documentPart type="bits">1010</tdml:documentPart><!-- subfield1 -->
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e2 xmlns:ex="http://example.com">
+ <ex:length>20</ex:length>
+ <ex:payload>
+ <ex:sublength>5</ex:sublength>
+ <ex:subfield1>21</ex:subfield1>
+ <ex:subfield2>10</ex:subfield2>
+ </ex:payload>
+ </ex:e2>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: alignmentPaddingOVC3
+ Schema: delimitedStrings
+ Purpose: This test demonstrates roundtrip of an element with padding, OVC, and alignment
+-->
+ <tdml:unparserTestCase name="alignmentPaddingOVC3" model="alignmentPaddingOVC" root="e3"
+ config="cfg_forceFileBuffering">
+ <tdml:document>
+ <tdml:documentPart type="text">x20x</tdml:documentPart><!-- length of below fields in bits -->
+ <tdml:documentPart type="bits">0101</tdml:documentPart><!-- subfield1 length -->
+ <tdml:documentPart type="bits">10101</tdml:documentPart><!-- subfield1 -->
+ <tdml:documentPart type="bits">0000000</tdml:documentPart><!-- alignment skip -->
+ <tdml:documentPart type="bits">1010</tdml:documentPart><!-- subfield1 -->
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e3 xmlns:ex="http://example.com">
+ <ex:length>20</ex:length>
+ <ex:payload>
+ <ex:sublength>5</ex:sublength>
+ <ex:subfield1>21</ex:subfield1>
+ <ex:subfield2>10</ex:subfield2>
+ </ex:payload>
+ </ex:e3>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:unparserTestCase>
+
+<!--
+ Test Name: alignmentPaddingOVC4
+ Schema: delimitedStrings
+ Purpose: This test demonstrates an SDE when explicit length is less than textOutputMinLength
+-->
+ <tdml:unparserTestCase name="alignmentPaddingOVC4" model="alignmentPaddingOVC"
+ roundTrip="none" config="cfg_forceFileBuffering">
+ <tdml:document>2AB;</tdml:document>
+ <tdml:warnings>
+ <tdml:warning>Schema Definition Warning</tdml:warning>
+ <tdml:warning>Explicit dfdl:length</tdml:warning>
+ <tdml:warning>out of range</tdml:warning>
+ <tdml:warning>textOutputMinLength</tdml:warning>
+ </tdml:warnings>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <ex:e4>
+ <ex:payload>AB</ex:payload>
+ </ex:e4>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:unparserTestCase>
+
+</tdml:testSuite>
diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section00/general/TestFileBuffering.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section00/general/TestFileBuffering.scala
new file mode 100644
index 0000000..24fe40a
--- /dev/null
+++ b/daffodil-test/src/test/scala/org/apache/daffodil/section00/general/TestFileBuffering.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.section00.general
+
+import org.junit.Test
+import org.junit.AfterClass
+import org.apache.daffodil.tdml.Runner
+
+object TestUnparserFileBuffering {
+ val testDir = "/org/apache/daffodil/section00/general/"
+ val runner = Runner(testDir, "testUnparserFileBuffering.tdml")
+
+ @AfterClass def shutDown {
+ runner.reset
+ }
+}
+class TestUnparserFileBuffering {
+
+ import TestUnparserFileBuffering._
+
+ @Test def test_puaInfosetChars_01() { runner.runOneTest("puaInfosetChars_01") }
+ @Test def test_puaInfosetChars_02() { runner.runOneTest("puaInfosetChars_02") }
+
+ @Test def test_unparseFixedLengthString01() { runner.runOneTest("unparseFixedLengthString01") }
+ @Test def test_unparseFixedLengthString02() { runner.runOneTest("unparseFixedLengthString02") }
+ @Test def test_unparseFixedLengthString03() { runner.runOneTest("unparseFixedLengthString03") }
+
+ @Test def test_parseFixedLengthString01() { runner.runOneTest("parseFixedLengthString01") }
+ @Test def test_parseFixedLengthStringLength0() { runner.runOneTest("parseFixedLengthStringLength0") }
+
+ @Test def test_negativeUnparseTest01() { runner.runOneTest("negativeUnparseTest01") }
+ @Test def test_negativeUnparseTest02() { runner.runOneTest("negativeUnparseTest02") }
+ @Test def test_negativeUnparseTest03() { runner.runOneTest("negativeUnparseTest03") }
+ @Test def test_negativeUnparseTest04() { runner.runOneTest("negativeUnparseTest04") }
+ @Test def test_negativeUnparseTest05() { runner.runOneTest("negativeUnparseTest05") }
+
+ @Test def test_unparseDelimitedString01() { runner.runOneTest("unparseDelimitedString01") }
+ @Test def test_unparseDelimitedString02() { runner.runOneTest("unparseDelimitedString02") }
+ @Test def test_unparseDelimitedString03() { runner.runOneTest("unparseDelimitedString03") }
+ @Test def test_unparseDelimitedString04() { runner.runOneTest("unparseDelimitedString04") }
+ @Test def test_unparseDelimitedString05() { runner.runOneTest("unparseDelimitedString05") }
+ @Test def test_unparseDelimitedString06() { runner.runOneTest("unparseDelimitedString06") }
+ @Test def test_unparseDelimitedString07() { runner.runOneTest("unparseDelimitedString07") }
+
+ @Test def test_parseDelimitedString01() { runner.runOneTest("parseDelimitedString01") }
+
+ // DFDL-1650
+ @Test def test_alignmentPaddingOVC1() { runner.runOneTest("alignmentPaddingOVC1") }
+ @Test def test_alignmentPaddingOVC2() { runner.runOneTest("alignmentPaddingOVC2") }
+ @Test def test_alignmentPaddingOVC3() { runner.runOneTest("alignmentPaddingOVC3") }
+ @Test def test_alignmentPaddingOVC4() { runner.runOneTest("alignmentPaddingOVC4") }
+
+}