You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by ja...@apache.org on 2018/01/30 09:05:24 UTC

[incubator-daffodil] branch master updated: Fixed bug in calculating bit length for delimited unparsing

This is an automated email from the ASF dual-hosted git repository.

jadams pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git


The following commit(s) were added to refs/heads/master by this push:
     new 8d9aada  Fixed bug in calculating bit length for delimited unparsing
8d9aada is described below

commit 8d9aadab07a2f422dfae527b3a4d1cb4047a6efa
Author: Josh Adams <ja...@tresys.com>
AuthorDate: Wed Jan 24 14:12:53 2018 -0500

    Fixed bug in calculating bit length for delimited unparsing
    
     - getBitLength now simply returns 0 for delimited unparsers
     - Refactored *FromBigInteger functions to be more clear in how the
     packed data is padded
     - Refactored delimited unparsers to not take a minLength parameter
     - Moved existing tests out of scala-debug that now work
     - Added several test cases for delimited parsing of packed numbers
     - Added some unparser test cases for delimited unparsing.
    
    DAFFODIL-1739
---
 .../grammar/primitives/PrimitivesLengthKind.scala  |  30 +--
 .../illinois/ncsa/daffodil/util/DecimalUtils.scala |  30 ++-
 .../processors/unparsers/BCDUnparsers.scala        |  18 +-
 .../unparsers/IBM4690PackedDecimalUnparsers.scala  |  18 +-
 .../unparsers/PackedBinaryUnparserTraits.scala     |  29 +--
 .../unparsers/PackedDecimalUnparsers.scala         |  18 +-
 .../processors/parsers/DelimitedParsers.scala      |   8 +-
 .../test-suite/tresys-contributed/AU.dfdl.xsd      |   8 +-
 .../test-suite/tresys-contributed/AU.tdml          |  20 +-
 .../edu/illinois/ncsa/daffodil/IBMTests2.scala     |   2 -
 .../edu/illinois/ncsa/daffodil/IBMTests.scala      |   9 +-
 .../edu/illinois/ncsa/daffodil/TresysTests3.scala  |   5 +
 .../ncsa/daffodil/section13/packed/packed.tdml     | 261 +++++++++++++++++++++
 .../daffodil/section13/packed/TestPacked.scala     |  13 +
 14 files changed, 356 insertions(+), 113 deletions(-)

diff --git a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/grammar/primitives/PrimitivesLengthKind.scala b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/grammar/primitives/PrimitivesLengthKind.scala
index 0ce9838..a270da1 100644
--- a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/grammar/primitives/PrimitivesLengthKind.scala
+++ b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/grammar/primitives/PrimitivesLengthKind.scala
@@ -55,12 +55,12 @@ import edu.illinois.ncsa.daffodil.processors.parsers.StringDelimitedParser
 import edu.illinois.ncsa.daffodil.processors.parsers.StringOfSpecifiedLengthParser
 import edu.illinois.ncsa.daffodil.processors.parsers.{ Parser => DaffodilParser }
 import edu.illinois.ncsa.daffodil.processors.unparsers.HexBinaryMinLengthInBytesUnparser
-import edu.illinois.ncsa.daffodil.processors.unparsers.PackedIntegerMinLengthInBytesUnparser
-import edu.illinois.ncsa.daffodil.processors.unparsers.PackedDecimalMinLengthInBytesUnparser
-import edu.illinois.ncsa.daffodil.processors.unparsers.BCDIntegerMinLengthInBytesUnparser
-import edu.illinois.ncsa.daffodil.processors.unparsers.BCDDecimalMinLengthInBytesUnparser
-import edu.illinois.ncsa.daffodil.processors.unparsers.IBM4690PackedIntegerMinLengthInBytesUnparser
-import edu.illinois.ncsa.daffodil.processors.unparsers.IBM4690PackedDecimalMinLengthInBytesUnparser
+import edu.illinois.ncsa.daffodil.processors.unparsers.PackedIntegerDelimitedUnparser
+import edu.illinois.ncsa.daffodil.processors.unparsers.PackedDecimalDelimitedUnparser
+import edu.illinois.ncsa.daffodil.processors.unparsers.BCDIntegerDelimitedUnparser
+import edu.illinois.ncsa.daffodil.processors.unparsers.BCDDecimalDelimitedUnparser
+import edu.illinois.ncsa.daffodil.processors.unparsers.IBM4690PackedIntegerDelimitedUnparser
+import edu.illinois.ncsa.daffodil.processors.unparsers.IBM4690PackedDecimalDelimitedUnparser
 import edu.illinois.ncsa.daffodil.processors.unparsers.HexBinarySpecifiedLengthUnparser
 import edu.illinois.ncsa.daffodil.processors.unparsers.LiteralNilDelimitedEndOfDataUnparser
 import edu.illinois.ncsa.daffodil.processors.unparsers.OptionalInfixSepUnparser
@@ -217,8 +217,7 @@ abstract class PackedIntegerDelimited(e: ElementBase, signed: Boolean, packedSig
     isDelimRequired,
     packedSignCodes)
 
-  override lazy val unparser: DaffodilUnparser = new PackedIntegerMinLengthInBytesUnparser(
-    e.minLength.intValue,
+  override lazy val unparser: DaffodilUnparser = new PackedIntegerDelimitedUnparser(
     e.elementRuntimeData,
     packedSignCodes)
 }
@@ -239,8 +238,7 @@ abstract class PackedDecimalDelimited(e: ElementBase, packedSignCodes: PackedSig
     e.binaryDecimalVirtualPoint,
     packedSignCodes)
 
-  override lazy val unparser: DaffodilUnparser = new PackedDecimalMinLengthInBytesUnparser(
-    e.minLength.intValue,
+  override lazy val unparser: DaffodilUnparser = new PackedDecimalDelimitedUnparser(
     e.elementRuntimeData,
     e.binaryDecimalVirtualPoint,
     packedSignCodes)
@@ -260,8 +258,7 @@ abstract class BCDIntegerDelimited(e: ElementBase)
     fieldDFAParseEv,
     isDelimRequired)
 
-  override lazy val unparser: DaffodilUnparser = new BCDIntegerMinLengthInBytesUnparser(
-    e.minLength.intValue,
+  override lazy val unparser: DaffodilUnparser = new BCDIntegerDelimitedUnparser(
     e.elementRuntimeData)
 }
 
@@ -280,8 +277,7 @@ abstract class BCDDecimalDelimited(e: ElementBase)
     isDelimRequired,
     e.binaryDecimalVirtualPoint)
 
-  override lazy val unparser: DaffodilUnparser = new BCDDecimalMinLengthInBytesUnparser(
-    e.minLength.intValue,
+  override lazy val unparser: DaffodilUnparser = new BCDDecimalDelimitedUnparser(
     e.elementRuntimeData,
     e.binaryDecimalVirtualPoint)
 }
@@ -300,8 +296,7 @@ abstract class IBM4690PackedIntegerDelimited(e: ElementBase, signed: Boolean)
     fieldDFAParseEv,
     isDelimRequired)
 
-  override lazy val unparser: DaffodilUnparser = new IBM4690PackedIntegerMinLengthInBytesUnparser(
-    e.minLength.intValue,
+  override lazy val unparser: DaffodilUnparser = new IBM4690PackedIntegerDelimitedUnparser(
     e.elementRuntimeData)
 }
 
@@ -320,8 +315,7 @@ abstract class IBM4690PackedDecimalDelimited(e: ElementBase)
     isDelimRequired,
     e.binaryDecimalVirtualPoint)
 
-  override lazy val unparser: DaffodilUnparser = new IBM4690PackedDecimalMinLengthInBytesUnparser(
-    e.minLength.intValue,
+  override lazy val unparser: DaffodilUnparser = new IBM4690PackedDecimalDelimitedUnparser(
     e.elementRuntimeData,
     e.binaryDecimalVirtualPoint)
 }
diff --git a/daffodil-lib/src/main/scala/edu/illinois/ncsa/daffodil/util/DecimalUtils.scala b/daffodil-lib/src/main/scala/edu/illinois/ncsa/daffodil/util/DecimalUtils.scala
index 194b898..48e57c3 100644
--- a/daffodil-lib/src/main/scala/edu/illinois/ncsa/daffodil/util/DecimalUtils.scala
+++ b/daffodil-lib/src/main/scala/edu/illinois/ncsa/daffodil/util/DecimalUtils.scala
@@ -111,15 +111,17 @@ object DecimalUtils {
     return new JBigDecimal(packedToBigInteger(num, signCodes), scale)
   }
 
-  def packedFromBigInteger(bigInt: JBigInteger, nBits: Int, signCodes: PackedSignCodes): Array[Byte] = {
+  def packedFromBigInteger(bigInt: JBigInteger, minLengthInBits: Int, signCodes: PackedSignCodes): Array[Byte] = {
     val negative = (bigInt.signum != 1)
     val inChars = bigInt.abs.toString.toCharArray
+    val numDigits = inChars.length
+    // Length required to fit all the digits of the number including sign nibble
+    val requiredBitLen = if (numDigits % 2 == 0) ((numDigits + 2) * 4) else ((numDigits + 1) * 4)
+    val bitLen = scala.math.max(minLengthInBits, requiredBitLen)
     var offset = 0
     var inPos = 0
-    // Don't count the '-' symbol when determing the number of digits
-    val numDigits = inChars.length
-    val outArray = new Array[Byte](nBits/8)
-    val leadingZeros = if (numDigits % 2 == 0) (nBits/4 - numDigits - 1) else (nBits/4 - numDigits)
+    val outArray = new Array[Byte](bitLen/8)
+    val leadingZeros = if (numDigits % 2 == 0) (bitLen/4 - numDigits - 1) else (bitLen/4 - numDigits)
 
     // Add leading double zeros if necessary
     while ((offset * 2) < (leadingZeros - 1)) {
@@ -183,12 +185,16 @@ object DecimalUtils {
     new JBigDecimal(bcdToBigInteger(bcdNum), scale)
   }
 
-  def bcdFromBigInteger(bigInt: JBigInteger, nBits: Int): Array[Byte] = {
+  def bcdFromBigInteger(bigInt: JBigInteger, minLengthInBits: Int): Array[Byte] = {
     val inChars = bigInt.toString.toCharArray
+    val numDigits = inChars.length
+    // Need to have an even number of digits to fill out a complete byte
+    val requiredBitLen = if (numDigits % 2 == 0) (numDigits * 4) else ((numDigits + 1) * 4)
+    val bitLen = scala.math.max(minLengthInBits, requiredBitLen)
     var offset = 0
     var inPos = 0
-    val outArray = new Array[Byte](nBits/8)
-    val leadingZeros = nBits/4 - inChars.length
+    val outArray = new Array[Byte](bitLen/8)
+    val leadingZeros = bitLen/4 - inChars.length
 
     // Add leading double zeros if necessary
     while ((offset * 2) < (leadingZeros - 1)) {
@@ -277,15 +283,17 @@ object DecimalUtils {
     new JBigDecimal(ibm4690ToBigInteger(num), scale)
   }
 
-  def ibm4690FromBigInteger(bigInt: JBigInteger, nBits: Int): Array[Byte] = {
+  def ibm4690FromBigInteger(bigInt: JBigInteger, minLengthInBits: Int): Array[Byte] = {
     val negative = (bigInt.signum != 1)
     val inChars = bigInt.abs.toString.toCharArray
     var wrote_negative = false
     var offset = 0
     var inPos = 0
     val numDigits = if (negative) inChars.length + 1 else inChars.length
-    val outArray = new Array[Byte](nBits/8)
-    val leadingZeros = if (numDigits % 2 == 0) (nBits/4 - numDigits) else (nBits/4 - (numDigits + 1))
+    val requiredBitLen = if (numDigits % 2 == 0) (numDigits * 4) else ((numDigits + 1) * 4)
+    val bitLen = scala.math.max(minLengthInBits, requiredBitLen)
+    val outArray = new Array[Byte](bitLen/8)
+    val leadingZeros = if (numDigits % 2 == 0) (bitLen/4 - numDigits) else (bitLen/4 - (numDigits + 1))
 
     // Add leading double zeros if necessary
     while ((offset * 2) < (leadingZeros - 1)) {
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/BCDUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/BCDUnparsers.scala
index 8457fb3..6c7d9fb 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/BCDUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/BCDUnparsers.scala
@@ -48,16 +48,11 @@ class BCDIntegerRuntimeLengthUnparser(
   override lazy val runtimeDependencies = List(lengthEv)
 }
 
-final class BCDIntegerMinLengthInBytesUnparser(
-  minLengthInBytes: Int,
+final class BCDIntegerDelimitedUnparser(
   e: ElementRuntimeData)
   extends BCDIntegerBaseUnparser(e) {
 
-  override def getBitLength(state: ParseOrUnparseState): Int = {
-    val len = state.currentNode.get.asSimple.dataValue.asInstanceOf[Array[Byte]].length * 8
-    val min = minLengthInBytes * 8
-    scala.math.max(len, min)
-  }
+  override def getBitLength(state: ParseOrUnparseState): Int = { 0 }
 }
 
 abstract class BCDDecimalBaseUnparser(
@@ -87,15 +82,10 @@ class BCDDecimalRuntimeLengthUnparser(
   override lazy val runtimeDependencies = List(lengthEv)
 }
 
-final class BCDDecimalMinLengthInBytesUnparser(
-  minLengthInBytes: Int,
+final class BCDDecimalDelimitedUnparser(
   e: ElementRuntimeData,
   binaryDecimalVirtualPoint: Int)
   extends BCDDecimalBaseUnparser(e, binaryDecimalVirtualPoint) {
 
-  override def getBitLength(state: ParseOrUnparseState): Int = {
-    val len = state.currentNode.get.asSimple.dataValue.asInstanceOf[Array[Byte]].length * 8
-    val min = minLengthInBytes * 8
-    scala.math.max(len, min)
-  }
+  override def getBitLength(state: ParseOrUnparseState): Int = { 0 }
 }
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/IBM4690PackedDecimalUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/IBM4690PackedDecimalUnparsers.scala
index 7f1f331..59e75bf 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/IBM4690PackedDecimalUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/IBM4690PackedDecimalUnparsers.scala
@@ -48,16 +48,11 @@ class IBM4690PackedIntegerRuntimeLengthUnparser(
   override lazy val runtimeDependencies = List(lengthEv)
 }
 
-final class IBM4690PackedIntegerMinLengthInBytesUnparser(
-  minLengthInBytes: Int,
+final class IBM4690PackedIntegerDelimitedUnparser(
   e: ElementRuntimeData)
   extends IBM4690PackedIntegerBaseUnparser(e) {
 
-  override def getBitLength(state: ParseOrUnparseState): Int = {
-    val len = state.currentNode.get.asSimple.dataValue.asInstanceOf[Array[Byte]].length * 8
-    val min = minLengthInBytes *  8
-    scala.math.max(len, min)
-  }
+  override def getBitLength(state: ParseOrUnparseState): Int = { 0 }
 }
 
 abstract class IBM4690PackedDecimalBaseUnparser(
@@ -87,15 +82,10 @@ class IBM4690PackedDecimalRuntimeLengthUnparser(
   override lazy val runtimeDependencies = List(lengthEv)
 }
 
-final class IBM4690PackedDecimalMinLengthInBytesUnparser(
-  minLengthInBytes: Int,
+final class IBM4690PackedDecimalDelimitedUnparser(
   e: ElementRuntimeData,
   binaryDecimalVirtualPoint: Int)
   extends IBM4690PackedDecimalBaseUnparser(e, binaryDecimalVirtualPoint) {
 
-  override def getBitLength(state: ParseOrUnparseState): Int = {
-    val len = state.currentNode.get.asSimple.dataValue.asInstanceOf[Array[Byte]].length * 8
-    val min = minLengthInBytes *  8
-    scala.math.max(len, min)
-  }
+  override def getBitLength(state: ParseOrUnparseState): Int = { 0 }
 }
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedBinaryUnparserTraits.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedBinaryUnparserTraits.scala
index fbcb6ec..e786949 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedBinaryUnparserTraits.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedBinaryUnparserTraits.scala
@@ -45,12 +45,7 @@ abstract class PackedBinaryBaseUnparser(
     val value = node.dataValue.asInstanceOf[JNumber]
     val dos = state.dataOutputStream
 
-    val res =
-      if (nBits > 0) {
-        putNumber(dos, value, nBits, state)
-      } else {
-        true
-      }
+    val res = putNumber(dos, value, nBits, state)
 
     if (!res) {
       Assert.invariant(dos.maybeRelBitLimit0b.isDefined)
@@ -72,11 +67,9 @@ abstract class PackedBinaryDecimalBaseUnparser(
     if (bigDec.movePointRight(binaryDecimalVirtualPoint).scale != 0)
       e.schemaDefinitionError("Decimal point of number '%s' does not match the binaryVirtualDecmialPoint: %d", bigDec, binaryDecimalVirtualPoint)
 
-    dos.putByteArray(
-      fromBigInteger(bigDec.unscaledValue, nBits),
-      nBits,
-      finfo)
-  }
+      val packedNum = fromBigInteger(bigDec.unscaledValue, nBits)
+      dos.putByteArray(packedNum, packedNum.length * 8, finfo)
+    }
 }
 
 abstract class PackedBinaryIntegerBaseUnparser(
@@ -85,14 +78,12 @@ abstract class PackedBinaryIntegerBaseUnparser(
 
   override def putNumber(dos: DataOutputStream, number: JNumber, nBits: Int, finfo: FormatInfo): Boolean = {
 
-    val bigInt = number.isInstanceOf[JBigInteger] match {
-      case true => number.asInstanceOf[JBigInteger]
-      case false => new JBigInteger(number.toString)
-    }
+      val bigInt = number.isInstanceOf[JBigInteger] match {
+        case true => number.asInstanceOf[JBigInteger]
+        case false => new JBigInteger(number.toString)
+      }
 
-    dos.putByteArray(
-      fromBigInteger(bigInt, nBits),
-      nBits,
-      finfo)
+      val packedNum = fromBigInteger(bigInt, nBits)
+      dos.putByteArray(packedNum, packedNum.length * 8, finfo)
   }
 }
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedDecimalUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedDecimalUnparsers.scala
index e09a6b5..789aac3 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedDecimalUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedDecimalUnparsers.scala
@@ -51,17 +51,12 @@ class PackedIntegerRuntimeLengthUnparser(
   override lazy val runtimeDependencies = List(lengthEv)
 }
 
-final class PackedIntegerMinLengthInBytesUnparser(
-  minLengthInBytes: Int,
+final class PackedIntegerDelimitedUnparser(
   e: ElementRuntimeData,
   packedSignCodes: PackedSignCodes)
   extends PackedIntegerBaseUnparser(e, packedSignCodes) {
 
-  override def getBitLength(state: ParseOrUnparseState): Int = {
-    val len = state.currentNode.get.asSimple.dataValue.asInstanceOf[Array[Byte]].length * 8
-    val min = minLengthInBytes *  8
-    scala.math.max(len, min)
-  }
+  override def getBitLength(state: ParseOrUnparseState): Int = { 0 }
 }
 
 
@@ -95,16 +90,11 @@ class PackedDecimalRuntimeLengthUnparser(
   override lazy val runtimeDependencies = List(lengthEv)
 }
 
-final class PackedDecimalMinLengthInBytesUnparser(
-  minLengthInBytes: Int,
+final class PackedDecimalDelimitedUnparser(
   e: ElementRuntimeData,
   binaryDecimalVirtualPoint: Int,
   packedSignCodes: PackedSignCodes)
   extends PackedDecimalBaseUnparser(e, binaryDecimalVirtualPoint, packedSignCodes) {
 
-  override def getBitLength(state: ParseOrUnparseState): Int = {
-    val len = state.currentNode.get.asSimple.dataValue.asInstanceOf[Array[Byte]].length * 8
-    val min = minLengthInBytes *  8
-    scala.math.max(len, min)
-  }
+  override def getBitLength(state: ParseOrUnparseState): Int = { 0 }
 }
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/DelimitedParsers.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/DelimitedParsers.scala
index 74da01f..f4a5829 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/DelimitedParsers.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/DelimitedParsers.scala
@@ -247,8 +247,8 @@ class IBM4690PackedIntegerDelimitedParser(
   isDelimRequired: Boolean)
   extends PackedBinaryIntegerDelimitedBaseParser(erd, textParser, fieldDFAEv, isDelimRequired) {
 
-  override def toBigInteger(num: Array[Byte]): JBigInteger = DecimalUtils.bcdToBigInteger(num)
-  override def toBigDecimal(num: Array[Byte], scale: Int): JBigDecimal = DecimalUtils.bcdToBigDecimal(num, scale)
+  override def toBigInteger(num: Array[Byte]): JBigInteger = DecimalUtils.ibm4690ToBigInteger(num)
+  override def toBigDecimal(num: Array[Byte], scale: Int): JBigDecimal = DecimalUtils.ibm4690ToBigDecimal(num, scale)
 
 }
 
@@ -260,7 +260,7 @@ class IBM4690PackedDecimalDelimitedParser(
   binaryDecimalVirtualPoint: Int)
   extends PackedBinaryDecimalDelimitedBaseParser(erd, textParser, fieldDFAEv, isDelimRequired, binaryDecimalVirtualPoint) {
 
-  override def toBigInteger(num: Array[Byte]): JBigInteger = DecimalUtils.bcdToBigInteger(num)
-  override def toBigDecimal(num: Array[Byte], scale: Int): JBigDecimal = DecimalUtils.bcdToBigDecimal(num, scale)
+  override def toBigInteger(num: Array[Byte]): JBigInteger = DecimalUtils.ibm4690ToBigInteger(num)
+  override def toBigDecimal(num: Array[Byte], scale: Int): JBigDecimal = DecimalUtils.ibm4690ToBigDecimal(num, scale)
 
 }
diff --git a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/AU.dfdl.xsd b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/AU.dfdl.xsd
index a7686af..ccadd1c 100644
--- a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/AU.dfdl.xsd
+++ b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/AU.dfdl.xsd
@@ -3,6 +3,8 @@
   xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/" xmlns:xsd="http://www.w3.org/2001/XMLSchema"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:tns="http://example.com">
 
+  <include schemaLocation="edu/illinois/ncsa/daffodil/xsd/built-in-formats.xsd"/>
+
   <annotation>
     <appinfo source="http://www.ogf.org/dfdl/">
       <dfdl:format lengthKind="delimited" separator=""
@@ -11,7 +13,8 @@
         separatorSuppressionPolicy="anyEmpty" separatorPosition="infix"
         documentFinalTerminatorCanBeMissing='yes' byteOrder="bigEndian"
         binaryNumberRep='binary' representation="text" occursCountKind='parsed'
-        lengthUnits="bytes" textTrimKind='none' escapeSchemeRef='' />
+        lengthUnits="bytes" textTrimKind='none' escapeSchemeRef=''
+        ref="tns:daffodilTest1" />
     </appinfo>
   </annotation>
 
@@ -21,11 +24,12 @@
 
   <complexType name="example1">
     <sequence>
-      <element name="bcd" type="xsd:int" minOccurs="4"
+      <element name="bcd" type="xsd:unsignedInt" minOccurs="4"
         maxOccurs="4" dfdl:representation="binary" dfdl:binaryNumberRep="bcd"
         dfdl:lengthKind="explicit" dfdl:length="2" />
       <element name="packed" type="xsd:int" minOccurs="4"
         maxOccurs="4" dfdl:representation="binary" dfdl:binaryNumberRep="packed"
+        dfdl:binaryPackedSignCodes="C B F C" dfdl:binaryNumberCheckPolicy="strict"
         dfdl:lengthKind="explicit" dfdl:length="2" />
       <element name="string" type="xsd:string"
         dfdl:representation="text" dfdl:lengthKind="explicit"
diff --git a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/AU.tdml b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/AU.tdml
index 6334105..6fcc842 100644
--- a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/AU.tdml
+++ b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/AU.tdml
@@ -7,23 +7,23 @@
     description="Binary values packed and bsd">
     <document>
       <documentPart type="byte"><![CDATA[
-    1700 2300 2700 4119 8c01 8b01 1c72 1b72 7542 636e 2068 666f 4e20 6d75 6562 7372
+    0017 0023 0027 1941 018c 018b 721c 721b 4275 6e63 6820 6f66 204e 756d 6265 7273
     ]]></documentPart>
     </document>
     <infoset>
       <dfdlInfoset>
         <list xmlns="http://www.example.org/example1" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
-          <bcd xsi:type="xsd:int">0017</bcd>
-          <bcd xsi:type="xsd:int">0023</bcd>
-          <bcd xsi:type="xsd:int">0027</bcd>
-          <bcd xsi:type="xsd:int">1941</bcd>
-          <packed xsi:type="xsd:int">018</packed>
-          <packed xsi:type="xsd:int">-018</packed>
-          <packed xsi:type="xsd:int">721</packed>
-          <packed xsi:type="xsd:int">-721</packed>
+          <bcd>17</bcd>
+          <bcd>23</bcd>
+          <bcd>27</bcd>
+          <bcd>1941</bcd>
+          <packed>18</packed>
+          <packed>-18</packed>
+          <packed>721</packed>
+          <packed>-721</packed>
           <string>Bunch of Numbers</string>
         </list>
       </dfdlInfoset>
     </infoset>
   </parserTestCase>
-</testSuite>
\ No newline at end of file
+</testSuite>
diff --git a/daffodil-test-ibm1/src/test/scala-debug/edu/illinois/ncsa/daffodil/IBMTests2.scala b/daffodil-test-ibm1/src/test/scala-debug/edu/illinois/ncsa/daffodil/IBMTests2.scala
index 1333377..9742ec1 100644
--- a/daffodil-test-ibm1/src/test/scala-debug/edu/illinois/ncsa/daffodil/IBMTests2.scala
+++ b/daffodil-test-ibm1/src/test/scala-debug/edu/illinois/ncsa/daffodil/IBMTests2.scala
@@ -59,8 +59,6 @@ class IBMTestsThatThrow {
   lazy val runner1 = new DFDLTestSuite(Misc.getRequiredResource(tdml1))
   lazy val runner2 = new DFDLTestSuite(Misc.getRequiredResource(tdml2))
 
-  @Test def test_scoping_define_format_8_02() { runner1.runOneTest("scoping_define_format_8_02") } //DFDL-565 - attributeFormDefault='qualified', packed
-  @Test def test_scoping_define_format_8_03() { runner1.runOneTest("scoping_define_format_8_03") } //DFDL-565 - attributeFormDefault='qualified', packed
   @Test def test_scoping_define_format_8_04() { runner1.runOneTest("scoping_define_format_8_04") } //DFDL-565 - attributeFormDefault='qualified'
 
   @Test def test_scoping_default_format_8_01() { runner1.runOneTest("scoping_default_format_8_01") } // DFDL-565 attributeFormDefault='qualified'
diff --git a/daffodil-test-ibm1/src/test/scala/edu/illinois/ncsa/daffodil/IBMTests.scala b/daffodil-test-ibm1/src/test/scala/edu/illinois/ncsa/daffodil/IBMTests.scala
index 85248dd..fdb3756 100644
--- a/daffodil-test-ibm1/src/test/scala/edu/illinois/ncsa/daffodil/IBMTests.scala
+++ b/daffodil-test-ibm1/src/test/scala/edu/illinois/ncsa/daffodil/IBMTests.scala
@@ -56,8 +56,6 @@ class IBMTestsThatPass {
   @Test def test_alignment_bytes_12_03() { runner1.runOneTest("alignment_bytes_12_03") }
   @Test def test_alignment_bytes_12_06() { runner1.runOneTest("alignment_bytes_12_06") }
 
-  @Test def test_scoping_define_format_8_05() { runner1.runOneTest("scoping_define_format_8_05") }
-
   @Test def test_length_delimited_12_01() { runner1.runOneTest("length_delimited_12_01") }
   @Test def test_length_delimited_12_04() { runner1.runOneTest("length_delimited_12_04") }
 
@@ -72,9 +70,10 @@ class IBMTestsThatPass {
 
   // Used to work, but now we get NYI for attributeFormDefault='qualified'
   // @Test def test_scoping_default_format_8_01() { runner1.runOneTest("scoping_default_format_8_01") }
-  // @Test def test_scoping_define_format_8_01() { runner1.runOneTest("scoping_define_format_8_01") }
-  @Test def test_scoping_define_format_8_02() { runner1.runOneTest("scoping_define_format_8_02") } //DFDL-565 - attributeFormDefault='qualified', packed
-  @Test def test_scoping_define_format_8_03() { runner1.runOneTest("scoping_define_format_8_03") } //DFDL-565 - attributeFormDefault='qualified', packed
+  @Test def test_scoping_define_format_8_02() { runner1.runOneTest("scoping_define_format_8_02") }
+  @Test def test_scoping_define_format_8_03() { runner1.runOneTest("scoping_define_format_8_03") }
+  @Test def test_scoping_define_format_8_05() { runner1.runOneTest("scoping_define_format_8_05") }
+
 
   @Test def test_encoding_11_01() { runner1.runOneTest("encoding_11_01") }
 
diff --git a/daffodil-test-ibm1/src/test/scala/edu/illinois/ncsa/daffodil/TresysTests3.scala b/daffodil-test-ibm1/src/test/scala/edu/illinois/ncsa/daffodil/TresysTests3.scala
index f12a885..dc9a238 100644
--- a/daffodil-test-ibm1/src/test/scala/edu/illinois/ncsa/daffodil/TresysTests3.scala
+++ b/daffodil-test-ibm1/src/test/scala/edu/illinois/ncsa/daffodil/TresysTests3.scala
@@ -49,6 +49,9 @@ object TresysTests3 {
   lazy val runnerAM = Runner(testDir, "AM.tdml", validateTDMLFile = true, validateDFDLSchemas = false,
     compileAllTopLevel = true)
 
+  lazy val runnerAU = Runner(testDir, "AU.tdml", validateTDMLFile = true, validateDFDLSchemas = false,
+    compileAllTopLevel = true)
+
   lazy val runnerBC = Runner(testDir, "BC.tdml")
   lazy val runnerBD = Runner(testDir, "BD.tdml")
 }
@@ -83,4 +86,6 @@ class TresysTests3 {
   @Test def test_AM000() { runnerAM.runOneTest("AM000") }
   @Test def test_AM001() { runnerAM.runOneTest("AM001") }
 
+  @Test def test_AU000() { runnerAU.runOneTest("AU000") } // packed and bcd
+
 }
diff --git a/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section13/packed/packed.tdml b/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section13/packed/packed.tdml
index af0888b..f100415 100644
--- a/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section13/packed/packed.tdml
+++ b/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section13/packed/packed.tdml
@@ -186,6 +186,68 @@
 
   </tdml:defineSchema>
 
+  <tdml:defineSchema name="s14">
+    <dfdl:format ref="ex:daffodilTest1" lengthKind="delimited" encoding="ISO-8859-1" occursCountKind="implicit"
+    textNumberCheckPolicy="strict" textNumberPadCharacter="0" textNumberJustification="right"
+    lengthUnits="bytes" binaryPackedSignCodes="C D F C" binaryNumberCheckPolicy="strict"
+    binaryDecimalVirtualPoint="2"/>
+
+  <xs:element name="packedIntSeq" dfdl:lengthKind="delimited">
+    <xs:complexType>
+      <xs:sequence dfdl:initiatedContent="no" dfdl:separatorPosition="infix"
+        dfdl:sequenceKind="ordered" dfdl:separator="*" dfdl:terminator=";">
+        <xs:element name="num" type="xs:int" dfdl:representation="binary" dfdl:binaryNumberRep="packed" maxOccurs="unbounded" minOccurs="0"/>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="packedDecSeq" dfdl:lengthKind="delimited" >
+    <xs:complexType>
+      <xs:sequence dfdl:initiatedContent="no" dfdl:separatorPosition="infix"
+        dfdl:sequenceKind="ordered" dfdl:separator="*" dfdl:terminator=";">
+        <xs:element name="num" type="xs:decimal" dfdl:representation="binary" dfdl:binaryNumberRep="packed" maxOccurs="unbounded" minOccurs="0"/>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="bcdIntSeq" dfdl:lengthKind="delimited" >
+    <xs:complexType>
+      <xs:sequence dfdl:initiatedContent="no" dfdl:separatorPosition="infix"
+        dfdl:sequenceKind="ordered" dfdl:separator="," dfdl:terminator=";">
+        <xs:element name="num" type="xs:unsignedInt" dfdl:representation="binary" dfdl:binaryNumberRep="bcd" maxOccurs="unbounded" minOccurs="0"/>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="bcdDecSeq" dfdl:lengthKind="delimited" >
+    <xs:complexType>
+      <xs:sequence dfdl:initiatedContent="no" dfdl:separatorPosition="infix"
+        dfdl:sequenceKind="ordered" dfdl:separator="," dfdl:terminator=";">
+        <xs:element name="num" type="xs:decimal" dfdl:representation="binary" dfdl:binaryNumberRep="bcd" maxOccurs="unbounded" minOccurs="0"/>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="ibmIntSeq" dfdl:lengthKind="delimited" >
+    <xs:complexType>
+      <xs:sequence dfdl:initiatedContent="no" dfdl:separatorPosition="infix"
+        dfdl:sequenceKind="ordered" dfdl:separator="," dfdl:terminator=";">
+        <xs:element name="num" type="xs:int" dfdl:representation="binary" dfdl:binaryNumberRep="ibm4690Packed" maxOccurs="unbounded" minOccurs="0"/>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="ibmDecSeq" dfdl:lengthKind="delimited" >
+    <xs:complexType>
+      <xs:sequence dfdl:initiatedContent="no" dfdl:separatorPosition="infix"
+        dfdl:sequenceKind="ordered" dfdl:separator="," dfdl:terminator=";">
+        <xs:element name="num" type="xs:decimal" dfdl:representation="binary" dfdl:binaryNumberRep="ibm4690Packed" maxOccurs="unbounded" minOccurs="0"/>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+
+  </tdml:defineSchema>
+
 
   <tdml:parserTestCase name="hexCharset01" root="int01" model="s1"
     description="hex works as a charset encoding">
@@ -648,5 +710,204 @@
     </tdml:errors>
   </tdml:parserTestCase>
 
+  <tdml:parserTestCase name="DelimitedPackedIntSeq" root="packedIntSeq" model="s14"
+    description="Parsing a delimited sequence of the various packed formats">
+
+    <tdml:document>
+      <tdml:documentPart type="byte">003C 2A 123D 3B</tdml:documentPart>
+    </tdml:document>
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <packedIntSeq>
+          <num>3</num>
+          <num>-123</num>
+        </packedIntSeq>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+  </tdml:parserTestCase>
+
+  <tdml:parserTestCase name="DelimitedPackedDecSeq" root="packedDecSeq" model="s14"
+    description="Parsing a delimited sequence of the various packed formats">
+
+    <tdml:document>
+      <tdml:documentPart type="byte">003C 2A 123D 3B</tdml:documentPart>
+    </tdml:document>
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <packedDecSeq>
+          <num>0.03</num>
+          <num>-1.23</num>
+        </packedDecSeq>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+  </tdml:parserTestCase>
+
+  <tdml:unparserTestCase name="DelimitedPackedIntSeqUnparser" root="packedIntSeq" model="s14"
+    description="Parsing a delimited sequence of the various packed formats">
+
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <packedIntSeq>
+          <num>3</num>
+          <num>-123</num>
+        </packedIntSeq>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+
+    <tdml:document>
+      <tdml:documentPart type="byte">3C 2A 123D 3B</tdml:documentPart>
+    </tdml:document>
+
+  </tdml:unparserTestCase>
+
+  <tdml:unparserTestCase name="DelimitedPackedDecSeqUnparser" root="packedDecSeq" model="s14"
+    description="Parsing a delimited sequence of the various packed formats">
+
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <packedDecSeq>
+          <num>0.03</num>
+          <num>-1.23</num>
+        </packedDecSeq>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+
+    <tdml:document>
+      <tdml:documentPart type="byte">3C 2A 123D 3B</tdml:documentPart>
+    </tdml:document>
+  </tdml:unparserTestCase>
+
+  <tdml:parserTestCase name="DelimitedBCDIntSeq" root="bcdIntSeq" model="s14"
+    description="Parsing a delimited sequence of the various bcd formats">
+
+    <tdml:document>
+      <tdml:documentPart type="byte">0003 2C 0123 3B</tdml:documentPart>
+    </tdml:document>
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <bcdIntSeq>
+          <num>3</num>
+          <num>123</num>
+        </bcdIntSeq>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+  </tdml:parserTestCase>
+
+  <tdml:parserTestCase name="DelimitedBCDDecSeq" root="bcdDecSeq" model="s14"
+    description="Parsing a delimited sequence of the various bcd formats">
+
+    <tdml:document>
+      <tdml:documentPart type="byte">0003 2C 0123 3B</tdml:documentPart>
+    </tdml:document>
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <bcdDecSeq>
+          <num>0.03</num>
+          <num>1.23</num>
+        </bcdDecSeq>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+  </tdml:parserTestCase>
+
+  <tdml:unparserTestCase name="DelimitedBCDIntSeqUnparser" root="bcdIntSeq" model="s14"
+    description="Parsing a delimited sequence of the various bcd formats">
+
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <bcdIntSeq>
+          <num>3</num>
+          <num>123</num>
+        </bcdIntSeq>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+
+    <tdml:document>
+      <tdml:documentPart type="byte">03 2C 0123 3B</tdml:documentPart>
+    </tdml:document>
+  </tdml:unparserTestCase>
+
+  <tdml:unparserTestCase name="DelimitedBCDDecSeqUnparser" root="bcdDecSeq" model="s14"
+    description="Parsing a delimited sequence of the various bcd formats">
+
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <bcdDecSeq>
+          <num>0.03</num>
+          <num>1.23</num>
+        </bcdDecSeq>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+
+    <tdml:document>
+      <tdml:documentPart type="byte">03 2C 0123 3B</tdml:documentPart>
+    </tdml:document>
+  </tdml:unparserTestCase>
+
+
+  <tdml:parserTestCase name="DelimitedIBM4690IntSeq" root="ibmIntSeq" model="s14"
+    description="Parsing a delimited sequence of the various ibm formats">
+
+    <tdml:document>
+      <tdml:documentPart type="byte">FFF3 2C D123 3B</tdml:documentPart>
+    </tdml:document>
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <ibmIntSeq>
+          <num>3</num>
+          <num>-123</num>
+        </ibmIntSeq>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+  </tdml:parserTestCase>
+
+  <tdml:parserTestCase name="DelimitedIBM4690DecSeq" root="ibmDecSeq" model="s14"
+    description="Parsing a delimited sequence of the various ibm formats">
+
+    <tdml:document>
+      <tdml:documentPart type="byte">FFF3 2C D123 3B</tdml:documentPart>
+    </tdml:document>
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <ibmDecSeq>
+          <num>0.03</num>
+          <num>-1.23</num>
+        </ibmDecSeq>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+  </tdml:parserTestCase>
+
+  <tdml:unparserTestCase name="DelimitedIBM4690IntSeqUnparser" root="ibmIntSeq" model="s14"
+    description="Parsing a delimited sequence of the various ibm formats">
+
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <ibmIntSeq>
+          <num>3</num>
+          <num>-123</num>
+        </ibmIntSeq>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+
+    <tdml:document>
+      <tdml:documentPart type="byte">F3 2C D123 3B</tdml:documentPart>
+    </tdml:document>
+  </tdml:unparserTestCase>
+
+  <tdml:unparserTestCase name="DelimitedIBM4690DecSeqUnparser" root="ibmDecSeq" model="s14"
+    description="Parsing a delimited sequence of the various ibm formats">
+
+    <tdml:infoset>
+      <tdml:dfdlInfoset>
+        <ibmDecSeq>
+          <num>0.03</num>
+          <num>-1.23</num>
+        </ibmDecSeq>
+      </tdml:dfdlInfoset>
+    </tdml:infoset>
+
+    <tdml:document>
+      <tdml:documentPart type="byte">F3 2C D123 3B</tdml:documentPart>
+    </tdml:document>
+  </tdml:unparserTestCase>
 
 </tdml:testSuite>
diff --git a/daffodil-test/src/test/scala/edu/illinois/ncsa/daffodil/section13/packed/TestPacked.scala b/daffodil-test/src/test/scala/edu/illinois/ncsa/daffodil/section13/packed/TestPacked.scala
index 8aa4a72..1b12e75 100644
--- a/daffodil-test/src/test/scala/edu/illinois/ncsa/daffodil/section13/packed/TestPacked.scala
+++ b/daffodil-test/src/test/scala/edu/illinois/ncsa/daffodil/section13/packed/TestPacked.scala
@@ -89,4 +89,17 @@ class TestPacked {
   @Test def testIBM4690Charset08(): Unit = { runner.runOneTest("IBM4690Charset08") }
   @Test def testIBM4690Charset09(): Unit = { runner.runOneTest("IBM4690Charset09") }
   @Test def testIBM4690Charset10(): Unit = { runner.runOneTest("IBM4690Charset10") }
+
+  @Test def testDelimitedPackedIntSeq(): Unit = { runner.runOneTest("DelimitedPackedIntSeq") }
+  @Test def testDelimitedPackedDecSeq(): Unit = { runner.runOneTest("DelimitedPackedDecSeq") }
+  @Test def testDelimitedPackedIntSeqUnparser(): Unit = { runner.runOneTest("DelimitedPackedIntSeqUnparser") }
+  @Test def testDelimitedPackedDecSeqUnparser(): Unit = { runner.runOneTest("DelimitedPackedDecSeqUnparser") }
+  @Test def testDelimitedBCDIntSeq(): Unit = { runner.runOneTest("DelimitedBCDIntSeq") }
+  @Test def testDelimitedBCDDecSeq(): Unit = { runner.runOneTest("DelimitedBCDDecSeq") }
+  @Test def testDelimitedBCDIntSeqUnparser(): Unit = { runner.runOneTest("DelimitedBCDIntSeqUnparser") }
+  @Test def testDelimitedBCDDecSeqUnparser(): Unit = { runner.runOneTest("DelimitedBCDDecSeqUnparser") }
+  @Test def testDelimitedIBM4690IntSeq(): Unit = { runner.runOneTest("DelimitedIBM4690IntSeq") }
+  @Test def testDelimitedIBM4690DecSeq(): Unit = { runner.runOneTest("DelimitedIBM4690DecSeq") }
+  @Test def testDelimitedIBM4690IntSeqUnparser(): Unit = { runner.runOneTest("DelimitedIBM4690IntSeqUnparser") }
+  @Test def testDelimitedIBM4690DecSeqUnparser(): Unit = { runner.runOneTest("DelimitedIBM4690DecSeqUnparser") }
 }

-- 
To stop receiving notification emails like this one, please contact
jadams@apache.org.