You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by mb...@apache.org on 2018/01/26 16:01:11 UTC
[incubator-daffodil] branch master updated: Performance
improvements around FormatInfo change.
This is an automated email from the ASF dual-hosted git repository.
mbeckerle pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git
The following commit(s) were added to refs/heads/master by this push:
new ee4177b Performance improvements around FormatInfo change.
ee4177b is described below
commit ee4177b4c9bf84af2af19d8c9f67f283e233105e
Author: mbeckerle <mb...@tresys.com>
AuthorDate: Fri Jan 12 15:27:30 2018 -0500
Performance improvements around FormatInfo change.
Flattened out EvaluatableBase since it was only used one place.
Simplified methods - e.g., apply() gone.
Inlined simple methods.
Combined CharsetEv and CheckCharsetEv together into one so only one
evaluation is needed.
Introduced BitsCharset which is NOT derived from java's
java.nio.charset.Charset because well that class has a lot of final
methods. We need to be able to create a proxy charset that delegates to
an existing java Charset.
Because of all the final methods in java Charset, there is no way to do
this.
So instead we have Daffodil's own BitsCharset which implements an API
very close to that of java's Charset. But BitsCharset is NOT derived
from java Charset - which is fine because we are the only ones calling
it anyway.
A BitsCharset implements the additional bits-oriented methods on the
BitsCharset and on the encoders and decoders it creates.
Added Methods to allow us to skip mandatory alignment calculations.
Improve performance by caching FormatInfo members in the PState/UState.
Caches are cleared every parse1 or unparse1 call. So the FormatInfo
members are computed only once every parse or unparse.
Merged DFDLCharset and BitsCharset.
Changed all use of java.nio.charset.Charset... to JavaCharset...
Merged NBitsWidthCharset with NonByteSizedCharset into NBitsWidth_BitsCharset
Refactored to avoid code duplication.
Removed redundant (and error prone) ParserObject and UnparserObject.
All parsers and unparsers now must explicitly define
runtimeDependencies. Inheriting an empty one, by mistake, is not possible,
though one could still inherit it from a base class for the particular kind
of parser/unparser.
Fixed bug where bitOrder change wasn't being checked.
Caused test test_packed5Bit6 to fail. Fixed. Test evolved, but now
solicits the kind of change error that was expected.
DAFFODIL-1876, DAFFODIL-1597, DAFFODIL-1001
---
.../ncsa/daffodil/dsom/RuntimePropertyMixins.scala | 7 -
.../ncsa/daffodil/dsom/TermEncodingMixin.scala | 8 +-
.../ncsa/daffodil/grammar/AlignedMixin.scala | 13 +-
.../grammar/primitives/PrimitivesFraming.scala | 2 +-
.../ncsa/daffodil/dsom/TestBinaryInput_01.scala | 12 +-
.../daffodil/io/ByteBufferDataInputStream.scala | 30 +-
.../daffodil/io/DataOutputStreamImplMixin.scala | 4 +-
.../illinois/ncsa/daffodil/io/DecoderWrapper.scala | 188 -----
.../scala/edu/illinois/ncsa/daffodil/io/Dump.scala | 12 +-
.../edu/illinois/ncsa/daffodil/io/FormatInfo.scala | 12 +-
.../ncsa/daffodil/io/NonByteSizeCharset.scala | 716 ------------------
.../io/StringDataInputStreamForUnparse.scala | 3 +-
.../daffodil/processors/charset/BitsCharset.scala | 313 ++++++++
.../daffodil/processors/charset/CharsetUtils.scala | 79 +-
.../processors/charset/DFDL5BitPackedDecoder.scala | 2 +-
.../charset/DaffodilCharsetProvider.scala | 16 +-
.../processors/charset/HexLSBF4BitDecoder.scala | 4 +-
.../processors/charset/NBitsWidthCharsetBase.scala | 835 ++++++++++++++++++++-
.../processors/charset/OctalLSBF3BitDecoder.scala | 4 +-
.../charset/USASCII6BitPackedDecoder.scala | 2 +-
.../charset/USASCII7BitPackedDecoder.scala | 2 +-
.../charset/X_DFDL_6_BIT_DFI_264_DUI_001.scala | 2 +-
.../daffodil/processors/charset/X_DFDL_BITS.scala | 4 +-
.../ncsa/daffodil/io/FormatInfoForUnitTest.scala | 22 +-
.../io/TestByteBufferDataInputStream.scala | 4 +-
.../io/TestByteBufferDataInputStream3Bit.scala | 5 +-
.../io/TestByteBufferDataInputStream7.scala | 5 +-
.../illinois/ncsa/daffodil/io/TestDecoder.scala | 6 +-
.../io/TestNonByteSizedCharsetDecoders1Bit.scala | 35 +-
.../io/TestNonByteSizedCharsetDecoders3Bit.scala | 36 +-
.../io/TestNonByteSizedCharsetEncoders1Bit.scala | 12 +-
.../io/TestNonByteSizedCharsetEncoders3Bit.scala | 11 +-
.../processors/charset/TestBitsCharset.scala | 166 ++++
.../edu/illinois/ncsa/daffodil/util/Misc.scala | 8 +-
.../processors/unparsers/BCDUnparsers.scala | 8 +-
.../unparsers/BinaryNumberUnparsers.scala | 18 +-
.../unparsers/ConvertTextNumberUnparser.scala | 10 +-
.../processors/unparsers/DelimitedUnparsers.scala | 8 +-
.../processors/unparsers/DelimiterUnparsers.scala | 8 +-
.../unparsers/ElementKindUnparsers.scala | 38 +-
.../processors/unparsers/ElementUnparser.scala | 11 +-
.../unparsers/ExpressionEvaluatingUnparsers.scala | 36 +-
.../processors/unparsers/FramingUnparsers.scala | 11 +-
.../unparsers/HexBinaryLengthUnparser.scala | 10 +-
.../unparsers/IBM4690PackedDecimalUnparsers.scala | 4 +-
.../unparsers/NilEmptyCombinatorUnparsers.scala | 16 +-
.../unparsers/OptionalInfixSepUnparser.scala | 4 +-
.../unparsers/PackedBinaryUnparserTraits.scala | 50 +-
.../unparsers/PackedDecimalUnparsers.scala | 4 +-
.../processors/unparsers/SpecifiedLength2.scala | 43 +-
.../unparsers/SpecifiedLengthUnparsers.scala | 6 +-
.../ncsa/daffodil/dsom/EncodingLattice.scala | 10 +-
.../ncsa/daffodil/infoset/InfosetImpl.scala | 14 +-
.../ncsa/daffodil/processors/Dynamic.scala | 2 +-
.../daffodil/processors/EncodingRuntimeData.scala | 45 +-
.../ncsa/daffodil/processors/EvByteOrder.scala | 26 +-
.../ncsa/daffodil/processors/EvEncoding.scala | 63 +-
.../ncsa/daffodil/processors/Evaluatable.scala | 250 +++---
.../ncsa/daffodil/processors/ProcessorBases.scala | 64 +-
.../daffodil/processors/ProcessorStateBases.scala | 178 +++--
.../daffodil/processors/SchemaSetRuntimeData.scala | 2 +-
.../daffodil/processors/SuspendableOperation.scala | 11 -
.../ncsa/daffodil/processors/dfa/Parser.scala | 2 +-
.../processors/dfa/TextDelimitedParser.scala | 2 +-
.../processors/dfa/TextPaddingParser.scala | 6 +-
.../ncsa/daffodil/processors/dfa/TextParser.scala | 2 +-
.../processors/parsers/AssertPatternParsers.scala | 7 +-
.../processors/parsers/BinaryNumberParsers.scala | 20 +-
.../processors/parsers/DelimitedParsers.scala | 14 +-
.../processors/parsers/ElementCombinator1.scala | 5 +-
.../processors/parsers/ElementKindParsers.scala | 29 +-
.../parsers/ExpressionEvaluatingParsers.scala | 26 +-
.../daffodil/processors/parsers/NilParsers.scala | 3 +-
.../parsers/OptionalInfixSepParser.scala | 4 +-
.../ncsa/daffodil/processors/parsers/PState.scala | 4 +
.../processors/parsers/PackedBinaryTraits.scala | 30 +-
.../ncsa/daffodil/processors/parsers/Parser.scala | 88 ++-
.../daffodil/processors/parsers/PrimParser.scala | 72 --
.../processors/parsers/PrimitivesTextNumber1.scala | 14 +-
.../daffodil/processors/parsers/RepParsers.scala | 10 +-
.../parsers/SpecifiedLengthParsers.scala | 6 +-
.../parsers/UnorderedSequenceParser.scala | 4 +-
.../processors/unparsers/RepUnparsers.scala | 13 +-
.../daffodil/processors/unparsers/UState.scala | 16 +-
.../daffodil/processors/unparsers/Unparser.scala | 74 +-
.../illinois/ncsa/daffodil/tdml/TDMLRunner.scala | 17 +-
.../daffodil/section05/simple_types/BitOrder.tdml | 5 +-
.../section05/simple_types/BitOrderInvalid.tdml | 2 +-
.../ContentFramingProps.tdml | 41 +-
.../simple_types/TestSimpleTypesDebug.scala | 4 +-
.../simple_types/TestSimpleTypesNew.scala | 6 +-
91 files changed, 2267 insertions(+), 1789 deletions(-)
diff --git a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/RuntimePropertyMixins.scala b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/RuntimePropertyMixins.scala
index 4d4fb6c..57ab849 100644
--- a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/RuntimePropertyMixins.scala
+++ b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/RuntimePropertyMixins.scala
@@ -78,7 +78,6 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.NilKind
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.TextTrimKind
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.TextPadKind
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.YesNo
-import edu.illinois.ncsa.daffodil.processors.CheckEncodingEv
/*
* These are the DFDL properties which can have their values come
@@ -131,12 +130,6 @@ trait TermRuntimeValuedPropertiesMixin
} else
Nope
- final lazy val checkEncodingEv = {
- val ev = new CheckEncodingEv(termRuntimeData, alignmentValueInBits, charsetEv)
- ev.compile()
- ev
- }
-
final lazy val maybeFillByteEv = {
if (optionFillByteRaw.isDefined) {
val ev = new FillByteEv(fillByte, charsetEv, termRuntimeData)
diff --git a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/TermEncodingMixin.scala b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/TermEncodingMixin.scala
index 2a88633..dbd381a 100644
--- a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/TermEncodingMixin.scala
+++ b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/TermEncodingMixin.scala
@@ -36,7 +36,6 @@ import edu.illinois.ncsa.daffodil.processors.EncodingRuntimeData
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.Representation
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.EncodingErrorPolicy
import edu.illinois.ncsa.daffodil.processors.KnownEncodingMixin
-import edu.illinois.ncsa.daffodil.io.NonByteSizeCharset
/**
* Captures concepts around dfdl:encoding property and Terms.
@@ -90,15 +89,12 @@ trait TermEncodingMixin extends KnownEncodingMixin { self: Term =>
"Character set encoding name US-ASCII-7-BIT-PACKED is deprecated." +
"Please update your DFDL schema to use the name X-DFDL-US-ASCII-7-BIT-PACKED.")
val cs = charsetEv.optConstant.get
- cs.charset match {
- case nbs: NonByteSizeCharset => 1
- case _ => 8
- }
+ cs.mandatoryBitAlignment
} else 8 // unknown encodings always assumed to be 8-bit aligned.
}
lazy val encodingInfo =
- new EncodingRuntimeData(termRuntimeData, charsetEv, checkEncodingEv, schemaFileLocation, optionUTF16Width, defaultEncodingErrorPolicy,
+ new EncodingRuntimeData(termRuntimeData, charsetEv, schemaFileLocation, optionUTF16Width, defaultEncodingErrorPolicy,
summaryEncoding, isKnownEncoding, isScannable, knownEncodingAlignmentInBits)
/**
diff --git a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/grammar/AlignedMixin.scala b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/grammar/AlignedMixin.scala
index eba81ac..d503306 100644
--- a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/grammar/AlignedMixin.scala
+++ b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/grammar/AlignedMixin.scala
@@ -39,8 +39,8 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.AlignmentUnits
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.LengthKind
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.LengthUnits
import edu.illinois.ncsa.daffodil.util.Math
-import edu.illinois.ncsa.daffodil.io.NonByteSizeCharset
import edu.illinois.ncsa.daffodil.dsom.Root
+import edu.illinois.ncsa.daffodil.processors.charset.NBitsWidth_BitsCharset
case class AlignmentMultipleOf(nBits: Long) {
def *(that: AlignmentMultipleOf) = AlignmentMultipleOf(Math.gcd(nBits, that.nBits))
@@ -69,7 +69,10 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
if (!isRepresented) {
true
} else {
- (priorAlignmentWithLeadingSkipApprox % alignmentApprox) == 0
+ val pa = priorAlignmentWithLeadingSkipApprox
+ val aa = alignmentApprox
+ val res = (pa % aa) == 0
+ res
}
}.value
@@ -225,7 +228,7 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
} else {
// this only occurs when lengthUnits="characters", but the
// charset does not have a fixed width. In that case, we know
- // it's not a NonByteSizeCharset, so the length must be a
+ // it's not a NBitsWidth charset, so the length must be a
// multiple of 8
LengthMultipleOf(8)
}
@@ -263,7 +266,9 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
val isByteLength = this match {
case mg: ModelGroup => mg.groupMembers.forall { _.isKnownToBeByteAlignedAndByteLength }
case eb: ElementBase => {
- val isSelfByteSizeEncoding = eb.charsetEv.optConstant.map { !_.charset.isInstanceOf[NonByteSizeCharset] }.getOrElse(false)
+ val isSelfByteSizeEncoding = eb.charsetEv.optConstant.map {
+ !_.isInstanceOf[NBitsWidth_BitsCharset]
+ }.getOrElse(false)
val isSelfByteLength =
if (eb.isComplexType && eb.lengthKind == LengthKind.Implicit) {
eb.complexType.group.isKnownToBeByteAlignedAndByteLength
diff --git a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/grammar/primitives/PrimitivesFraming.scala b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/grammar/primitives/PrimitivesFraming.scala
index e72b9b3..7c00f96 100644
--- a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/grammar/primitives/PrimitivesFraming.scala
+++ b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/grammar/primitives/PrimitivesFraming.scala
@@ -72,7 +72,7 @@ case class TrailingSkipRegion(e: Term) extends SkipRegion(e, e.trailingSkipInBit
case class AlignmentFill(e: Term) extends Terminal(e, !e.isKnownToBeAligned) {
- private val alignment = e.alignmentValueInBits
+ private lazy val alignment = e.alignmentValueInBits // must be lazy, else guard can't "leave out" this term and then checks that are irrelevant will be done.
lazy val parser: Parser = new AlignmentFillParser(alignment, e.termRuntimeData)
lazy val unparser: Unparser = new AlignmentFillUnparser(alignment, e.termRuntimeData)
diff --git a/daffodil-core/src/test/scala/edu/illinois/ncsa/daffodil/dsom/TestBinaryInput_01.scala b/daffodil-core/src/test/scala/edu/illinois/ncsa/daffodil/dsom/TestBinaryInput_01.scala
index b0a316d..b961431 100644
--- a/daffodil-core/src/test/scala/edu/illinois/ncsa/daffodil/dsom/TestBinaryInput_01.scala
+++ b/daffodil-core/src/test/scala/edu/illinois/ncsa/daffodil/dsom/TestBinaryInput_01.scala
@@ -42,23 +42,23 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.ByteOrder
import java.nio.ByteBuffer
import edu.illinois.ncsa.daffodil.io.DataInputStream
import edu.illinois.ncsa.daffodil.io.FormatInfo
-import java.nio.charset.CharsetEncoder
-import java.nio.charset.CharsetDecoder
import edu.illinois.ncsa.daffodil.util.Maybe
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BinaryFloatRep
import edu.illinois.ncsa.daffodil.util.MaybeInt
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.EncodingErrorPolicy
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.UTF16Width
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharsetDecoder
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharsetEncoder
// Do no harm number 16 of 626 fail in regression, 154 in total of 797
class TestBinaryInput_01 {
class FakeFormatInfo(val bitOrder: BitOrder, val byteOrder: ByteOrder) extends FormatInfo {
- def encoder: CharsetEncoder = ???
- def decoder: CharsetDecoder = ???
- def reportingDecoder: CharsetDecoder = ???
- def replacingDecoder: CharsetDecoder = ???
+ def encoder: BitsCharsetEncoder = ???
+ def decoder: BitsCharsetDecoder = ???
+ def reportingDecoder: BitsCharsetDecoder = ???
+ def replacingDecoder: BitsCharsetDecoder = ???
def fillByte: Byte = ???
def binaryFloatRep: BinaryFloatRep = ???
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/ByteBufferDataInputStream.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/ByteBufferDataInputStream.scala
index ebf42af..74981fa 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/ByteBufferDataInputStream.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/ByteBufferDataInputStream.scala
@@ -37,8 +37,6 @@ import edu.illinois.ncsa.daffodil.util.MaybeULong
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.UTF16Width
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.ByteOrder
-import java.nio.charset.CharsetDecoder
-import java.nio.charset.StandardCharsets
import java.nio.charset.CodingErrorAction
import org.apache.commons.io.IOUtils
import java.io.ByteArrayOutputStream
@@ -56,6 +54,9 @@ import edu.illinois.ncsa.daffodil.util.Pool
import edu.illinois.ncsa.daffodil.util.MStackOf
import edu.illinois.ncsa.daffodil.api.DataStreamLimits
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.EncodingErrorPolicy
+import edu.illinois.ncsa.daffodil.processors.charset.NBitsWidth_BitsCharsetDecoder
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharsetDecoder
+import edu.illinois.ncsa.daffodil.processors.charset.StandardBitsCharsets
/**
* Factory for creating this type of DataInputStream
@@ -278,9 +279,9 @@ final class ByteBufferDataInputStream private (var data: ByteBuffer, initialBitP
override final def cst = st
- private def bytePos0b_ = data.position
+ @inline private def bytePos0b_ = data.position
- def bitPos0b: Long = (bytePos0b_ << 3) + st.bitOffset0b
+ @inline final def bitPos0b: Long = (bytePos0b_ << 3) + st.bitOffset0b
def setBitPos0b(newBitPos0b: Long) {
// threadCheck()
@@ -407,7 +408,7 @@ final class ByteBufferDataInputStream private (var data: ByteBuffer, initialBitP
var priorByte = Bits.asUnsignedByte(data.get())
var i = 0
- @inline
+ @inline // see comment below as to why this giant method is marked inline.
def addFragmentByte() = {
// This function is used at either the beginning or end of this function
// to read a fragement byte and store it in the correct location in the
@@ -738,7 +739,7 @@ final class ByteBufferDataInputStream private (var data: ByteBuffer, initialBitP
}
}
- def decodeIt(decoder: CharsetDecoder, cb: java.nio.CharBuffer, finfo: FormatInfo): CoderResult = {
+ private def decodeIt(decoder: BitsCharsetDecoder, cb: java.nio.CharBuffer, finfo: FormatInfo): CoderResult = {
var cr: CoderResult = null
var nCharsTransferred: Int = 0
var nBytesConsumed: Int = 0
@@ -746,7 +747,6 @@ final class ByteBufferDataInputStream private (var data: ByteBuffer, initialBitP
val bbRemainingBefore = data.remaining()
Assert.usage(cbRemainingBefore > 0)
-
cr = decoder.decode(data, cb, true)
nCharsTransferred = cbRemainingBefore - cb.remaining()
nBytesConsumed = bbRemainingBefore - data.remaining()
@@ -758,7 +758,7 @@ final class ByteBufferDataInputStream private (var data: ByteBuffer, initialBitP
//
if (cbRemainingBefore == 1 && nCharsTransferred == 0 && nBytesConsumed == 0 && cr.isOverflow()) {
Assert.invariant(bbRemainingBefore >= 4)
- Assert.invariant(finfo.decoder.charset() == StandardCharsets.UTF_8)
+ Assert.invariant(finfo.decoder.bitsCharset == StandardBitsCharsets.UTF_8)
val firstByte = data.get(data.position())
Assert.invariant(firstByte == 0xF0.toByte) // F0 means 3 more bytes for a total of 4
//
@@ -820,12 +820,16 @@ final class ByteBufferDataInputStream private (var data: ByteBuffer, initialBitP
def fillCharBuffer(cb: java.nio.CharBuffer, finfo: FormatInfo): MaybeULong = {
// threadCheck()
- if (!align(finfo.encodingMandatoryAlignmentInBits, finfo)) return MaybeULong.Nope
-
+ val dec = finfo.reportingDecoder
+ if (dec.isReset) {
+ // checks we can do once per reset
+ if (!align(finfo.encodingMandatoryAlignmentInBits, finfo)) return MaybeULong.Nope
+ finfo.bitOrder // just asking for the bit order does checking that it's ok.
+ }
//
// Corner case stuff for utf-8 and surrogate pairs.
//
- if (finfo.decoder.charset() == StandardCharsets.UTF_8) {
+ if (finfo.decoder.bitsCharset == StandardBitsCharsets.UTF_8) {
if (st.maybeTrailingSurrogateForUTF8.isDefined &&
st.priorBitPos == bitPos0b) {
// We're utf-8, the prior character was a leading surrogate,
@@ -846,8 +850,8 @@ final class ByteBufferDataInputStream private (var data: ByteBuffer, initialBitP
var nCharsTransferred: Int = 0
var nBytesConsumed: Int = 0
finfo.reportingDecoder.reset()
- val decoder = finfo.reportingDecoder match {
- case decoderWithBits: NonByteSizeCharsetDecoder => {
+ val decoder = dec match {
+ case decoderWithBits: NBitsWidth_BitsCharsetDecoder => {
decoderWithBits.setInitialBitOffset(st.bitOffset0b)
decoderWithBits.setFinalByteBitLimitOffset0b(st.maybeBitLimitOffset0b)
decoderWithBits
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/DataOutputStreamImplMixin.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/DataOutputStreamImplMixin.scala
index 10190a7..44784cb 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/DataOutputStreamImplMixin.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/DataOutputStreamImplMixin.scala
@@ -48,6 +48,7 @@ import edu.illinois.ncsa.daffodil.util.MaybeULong
import edu.illinois.ncsa.daffodil.equality._
import edu.illinois.ncsa.daffodil.util.Bits
import edu.illinois.ncsa.daffodil.util.LogLevel
+import edu.illinois.ncsa.daffodil.processors.charset.NBitsWidth_BitsCharsetEncoder
sealed trait DOSState
private[io] case object Active extends DOSState
@@ -667,7 +668,8 @@ trait DataOutputStreamImplMixin extends DataStreamCommonState
bb.flip
val bitsToWrite = finfo.encoder match {
- case encoderWithBits: NonByteSizeCharsetEncoder => encoderWithBits.bitWidthOfACodeUnit * nToTransfer
+ case encoderWithBits: NBitsWidth_BitsCharsetEncoder =>
+ encoderWithBits.bitsCharset.bitWidthOfACodeUnit * nToTransfer
case _ => bb.remaining * 8
}
if (putBitBuffer(bb, bitsToWrite, finfo) == 0) 0 else nToTransfer
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/DecoderWrapper.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/DecoderWrapper.scala
deleted file mode 100644
index 9d83b0b..0000000
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/DecoderWrapper.scala
+++ /dev/null
@@ -1,188 +0,0 @@
-/* Copyright (c) 2016 Tresys Technology, LLC. All rights reserved.
- *
- * Developed by: Tresys Technology, LLC
- * http://www.tresys.com
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy of
- * this software and associated documentation files (the "Software"), to deal with
- * the Software without restriction, including without limitation the rights to
- * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
- * of the Software, and to permit persons to whom the Software is furnished to do
- * so, subject to the following conditions:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimers.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimers in the
- * documentation and/or other materials provided with the distribution.
- *
- * 3. Neither the names of Tresys Technology, nor the names of its contributors
- * may be used to endorse or promote products derived from this Software
- * without specific prior written permission.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
- * SOFTWARE.
- */
-
-//
-// KEEP THIS FILE IN CASE WE HAVE TO GO BACK TO SUPPORTING JAVA 7
-//
-//package edu.illinois.ncsa.daffodil.io
-//
-//import java.nio.charset.Charset
-//import java.nio.charset.CodingErrorAction
-//import java.nio.ByteBuffer
-//import java.nio.CharBuffer
-//import java.nio.charset.CoderResult
-//import java.nio.charset.CharsetDecoder
-//import edu.illinois.ncsa.daffodil.exceptions.Assert
-//
-//object DecoderWrapper {
-//
-// lazy val hasJava7DecoderBug = {
-// val decoder = Charset.forName("utf-8").newDecoder()
-// val bb = ByteBuffer.allocate(6)
-// bb.put(-16.toByte) // invalid first utf-8 byte
-// bb.limit(6).position(0)
-// val cb = CharBuffer.allocate(1)
-// val cr = decoder.decode(bb, cb, true)
-// if (cr.isOverflow &&
-// cb.position == 0 &&
-// bb.position == 0) true
-// else if (cr.isError) false
-// else if (cr.isOverflow &&
-// cb.position == 1 &&
-// bb.position == 1 &&
-// cb.get(0) == this.unicodeReplacementChar) false
-// else
-// Assert.invariantFailed("Unexpected decoder behavior. " + cr)
-// }
-//
-// val unicodeReplacementChar = '\uFFFD'
-//
-//}
-//
-//final case class DecoderWrapper(val decoder: CharsetDecoder) {
-//
-// val unicodeReplacementChar = DecoderWrapper.unicodeReplacementChar
-//
-// private val initialDecoderOnMalformedInput = decoder.malformedInputAction()
-// private val initialDecoderOnUnmappable = decoder.unmappableCharacterAction()
-//
-// private val hasCodingErrorActionReport: Boolean =
-// initialDecoderOnMalformedInput == CodingErrorAction.REPORT &&
-// initialDecoderOnUnmappable == CodingErrorAction.REPORT
-//
-// private val cb2 = CharBuffer.allocate(2)
-// /**
-// * This decode method implements workaround for the
-// * problem in Java 7 when the CharBuffer.remaining == 1.
-// */
-// def decode(bb: ByteBuffer, cb: CharBuffer, noMoreData: Boolean): CoderResult = {
-// if (!DecoderWrapper.hasJava7DecoderBug) return decoder.decode(bb, cb, noMoreData)
-// if (cb.remaining > 1) return decoder.decode(bb, cb, noMoreData)
-// if (cb.remaining == 0) return CoderResult.OVERFLOW
-// //
-// // So the rest of this is for the case where the cb has exactly room for 1 character
-// Assert.invariant(cb.remaining == 1)
-//
-// val initialBBPosition = bb.position
-// val initialCBPosition = cb.position
-//
-// val decodeCR =
-// if (hasCodingErrorActionReport) decoder.decode(bb, cb, noMoreData) // avoid the try/catch etc.
-// else
-// try {
-// // save and restore the coding error actions
-// decoder.onMalformedInput(CodingErrorAction.REPORT)
-// decoder.onUnmappableCharacter(CodingErrorAction.REPORT)
-// val res1 = decoder.decode(bb, cb, noMoreData)
-//
-// if (res1.isError) {
-// // a malformed or unmappable - but this might not be the very first character.
-// if (cb.position() > initialCBPosition) {
-// // We decoded some characters. Let's return those successfully, as
-// // perhaps the caller doesn't need more than this many.
-// return CoderResult.OVERFLOW
-// }
-// // the very first character decode caused an error
-// if (hasCodingErrorActionReport) return res1
-// // fake an overflow that produced a replacement character.
-// val nBytes = res1.length
-// bb.position(initialBBPosition + nBytes)
-// cb.position(initialCBPosition + 1)
-// cb.put(initialCBPosition, unicodeReplacementChar)
-// return CoderResult.OVERFLOW
-// }
-// Assert.invariant(!res1.isError)
-// // must be an overflow or underflow
-// // Did we get a character, ie., decode successfully?
-// if (cb.position == initialCBPosition + 1)
-// return res1 // no decode error occurred. Normal.
-//
-// // Now we have to work-around the Java 7 bug.
-// // if the cb has only 1 location remaining,
-// // then on decode errors, java 7 doesn't report.
-// // (At least for the utf-8 decoder.)
-// // Instead you get an overflow, but no character
-// // was created and no bytes consumed
-// Assert.invariant(cb.position == initialCBPosition)
-// // got no character even though we overflowed
-// Assert.invariant(bb.position == initialBBPosition) // and consumed no bytes
-// // So we have to have a char buffer with remaining == 2
-// // use our own
-// cb2.clear
-// // scala bug. If instead of this val, you assign
-// // directly to the decodeCR, then the invariant test after fails
-// // as if the assignment never took place.
-// val res2 = decoder.decode(bb, cb2, noMoreData)
-// Assert.invariant(res2.isError)
-// Assert.invariant(cb2.position == 0)
-// Assert.invariant(bb.position == initialBBPosition)
-// //
-// // now we should have a malformed/unmapped exception
-// // that we can handle normally.
-// //
-// // either malformed or unmappable.
-// res2
-// } finally {
-// decoder.onMalformedInput(initialDecoderOnMalformedInput)
-// decoder.onUnmappableCharacter(initialDecoderOnUnmappable)
-// }
-// def doReplace = {
-// val nMalformedBytes = decodeCR.length
-// bb.position(initialBBPosition + nMalformedBytes)
-// cb.put(unicodeReplacementChar) // into our original cb.
-// CoderResult.OVERFLOW
-// }
-// def doIgnore = {
-// Assert.usageError("unsupported CodingErrorAction.IGNORE")
-// }
-//
-// if (decodeCR.isMalformed) initialDecoderOnMalformedInput match {
-// case CodingErrorAction.REPLACE => return doReplace
-// case CodingErrorAction.REPORT => return decodeCR
-// case CodingErrorAction.IGNORE => return doIgnore
-// }
-// //
-// // if we get here then it must be an unmappable character
-// //
-// Assert.invariant(decodeCR.isUnmappable)
-// initialDecoderOnMalformedInput match {
-// case CodingErrorAction.REPLACE => return doReplace
-// case CodingErrorAction.REPORT => return decodeCR
-// case CodingErrorAction.IGNORE => return doIgnore
-// }
-// Assert.impossible("should be no fall through to here.")
-// }
-//
-// def flush(cb: CharBuffer): CoderResult = decoder.flush(cb)
-//
-// def charset() = decoder.charset()
-//}
\ No newline at end of file
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/Dump.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/Dump.scala
index a87819e..c95b11c 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/Dump.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/Dump.scala
@@ -34,14 +34,14 @@ package edu.illinois.ncsa.daffodil.io
import java.nio.ByteBuffer
import java.nio.CharBuffer
-import java.nio.charset.Charset
import java.nio.charset.CodingErrorAction
import edu.illinois.ncsa.daffodil.exceptions.Assert
-import java.nio.charset.CharsetDecoder
import com.ibm.icu.lang.UCharacter
import com.ibm.icu.lang.UProperty
import edu.illinois.ncsa.daffodil.util.Misc
import edu.illinois.ncsa.daffodil.equality._
+import java.nio.charset.{ CharsetDecoder => JavaCharsetDecoder }
+import java.nio.charset.{ Charset => JavaCharset }
/**
* Hex/Bits and text dump formats for debug/trace purposes.
@@ -156,7 +156,7 @@ class DataDumper {
var nPadBytesFromPriorLine = 0
private def textDump(addr: Long, rowStart0b: Int, txtsb: StringBuilder,
- limit0b: Int, endByteAddress0b: Long, byteSource: ByteSource, decoder: Option[CharsetDecoder],
+ limit0b: Int, endByteAddress0b: Long, byteSource: ByteSource, decoder: Option[JavaCharsetDecoder],
textByteWidth: Int) {
var i = rowStart0b + nPadBytesFromPriorLine
txtsb ++= paddingFromPriorLine
@@ -451,8 +451,8 @@ class DataDumper {
}
}
- private def getReplacingDecoder(optEncodingName: Option[String]): Option[CharsetDecoder] = {
- val cs = optEncodingName.map { Charset.forName(_) }
+ private def getReplacingDecoder(optEncodingName: Option[String]): Option[JavaCharsetDecoder] = {
+ val cs = optEncodingName.map { JavaCharset.forName(_) }
lazy val decoder = cs.map { _.newDecoder() }
decoder foreach { d =>
d.onMalformedInput(CodingErrorAction.REPLACE)
@@ -467,7 +467,7 @@ class DataDumper {
private def convertToChar(startingBytePos0b: Long,
endingBytePos0b: Long,
bs: ByteSource,
- decoder: Option[CharsetDecoder]): (Char, Int, Int) = {
+ decoder: Option[JavaCharsetDecoder]): (Char, Int, Int) = {
Assert.invariant(decoder.map { d => Misc.isAsciiBased(d.charset()) }.getOrElse(true))
decoder match {
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/FormatInfo.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/FormatInfo.scala
index 4d07a76..fca78e6 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/FormatInfo.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/FormatInfo.scala
@@ -14,8 +14,6 @@
package edu.illinois.ncsa.daffodil.io
-import java.nio.charset.CharsetDecoder
-import java.nio.charset.CharsetEncoder
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.ByteOrder
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BinaryFloatRep
@@ -23,6 +21,8 @@ import edu.illinois.ncsa.daffodil.util.MaybeInt
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.EncodingErrorPolicy
import edu.illinois.ncsa.daffodil.util.Maybe
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.UTF16Width
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharsetDecoder
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharsetEncoder
/**
* Abstract interface to obtain format properties or values derived from
@@ -41,26 +41,26 @@ trait FormatInfo {
* `dfdl:encodingErrorPolicy`. This is the same as either the `reportingEncoder`
* or the `replacingEncoder`.
*/
- def encoder: CharsetEncoder
+ def encoder: BitsCharsetEncoder
/**
* Returns a charset decoder for this encoding configured for the
* `dfdl:encodingErrorPolicy`. This is the same as either the `reportingDecoder`
* or the `replacingDecoder`.
*/
- def decoder: CharsetDecoder
+ def decoder: BitsCharsetDecoder
/**
* Returns a decoder configured for `dfdl:encodingErrorPolicy="report"`
* regardless of the value of that property.
*/
- def reportingDecoder: CharsetDecoder
+ def reportingDecoder: BitsCharsetDecoder
/**
* Returns decoder configured for `dfdl:encodingErrorPolicy="replace"`
* regardless of the value of that property.
*/
- def replacingDecoder: CharsetDecoder
+ def replacingDecoder: BitsCharsetDecoder
/**
* Returns true if encoding is fixed width meaning has the
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/NonByteSizeCharset.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/NonByteSizeCharset.scala
deleted file mode 100644
index bcc6749..0000000
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/NonByteSizeCharset.scala
+++ /dev/null
@@ -1,716 +0,0 @@
-/* Copyright (c) 2016 Tresys Technology, LLC. All rights reserved.
- *
- * Developed by: Tresys Technology, LLC
- * http://www.tresys.com
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy of
- * this software and associated documentation files (the "Software"), to deal with
- * the Software without restriction, including without limitation the rights to
- * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
- * of the Software, and to permit persons to whom the Software is furnished to do
- * so, subject to the following conditions:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimers.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimers in the
- * documentation and/or other materials provided with the distribution.
- *
- * 3. Neither the names of Tresys Technology, nor the names of its contributors
- * may be used to endorse or promote products derived from this Software
- * without specific prior written permission.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
- * SOFTWARE.
- */
-
-package edu.illinois.ncsa.daffodil.io
-
-import edu.illinois.ncsa.daffodil.util.MaybeULong
-import edu.illinois.ncsa.daffodil.exceptions.Assert
-import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
-import java.nio.ByteBuffer
-import java.nio.CharBuffer
-import edu.illinois.ncsa.daffodil.util.Bits
-import java.nio.charset.CoderResult
-import java.nio.charset.CodingErrorAction
-import edu.illinois.ncsa.daffodil.util.MaybeInt
-import edu.illinois.ncsa.daffodil.util.Maybe
-
-/**
- * By "non byte sized" we mean some number of bits less than 8.
- */
-trait NonByteSizeCharset {
- def bitWidthOfACodeUnit: Int // in units of bits
- def requiredBitOrder: BitOrder
-
- protected def checks() {}
-
- protected final lazy val init = {
- checks()
- }
-}
-
-/**
- * Mixin for Decoders for Charsets which support initial bit offsets so that
- * their character codepoints need not be byte-aligned.
- */
-trait NonByteSizeCharsetDecoder
- extends NonByteSizeCharset { self: java.nio.charset.CharsetDecoder =>
-
- private var startBitOffset0b = 0
- private var startBitOffsetHasBeenSet = false
- private var startBitOffsetHasBeenUsed = false
-
- /**
- * Stores number of bits in the fragment byte at the end of the data.
- *
- * The frag byte value is an additional byte 1 past the end of the
- * byte buffer's last active position. That is, it is at the limit location.
- *
- * This limit location must exist. I.e., the limit must be less than the capacity.
- */
- private var maybeBitLimitOffset0b: MaybeULong = MaybeULong.Nope
- private var priorByte: MaybeInt = MaybeInt.Nope
- private var fragByte_ : MaybeInt = MaybeInt.Nope // populated from bytebuffer's final fragment byte if there is one.
-
- protected override def implReset() {
- resetStartBit()
- maybeBitLimitOffset0b = MaybeULong.Nope
- priorByte = MaybeInt.Nope
- priorByteBitCount = 0
- fragByte_ = MaybeInt.Nope
- }
-
- private def getFragByte(in: ByteBuffer): Int = {
- Assert.usage(this.maybeBitLimitOffset0b.isDefined &&
- this.maybeBitLimitOffset0b.get > 0)
- Assert.usage(in.limit() < in.capacity()) // room for 1 more byte.
- Assert.usage(in.remaining() == 0) // only call this after consuming all bytes.
- if (fragByte_.isEmpty) {
- val savedLimit = in.limit()
- in.limit(savedLimit + 1)
- val finalByte = in.get(savedLimit)
- in.limit(savedLimit)
- fragByte_ = MaybeInt(finalByte)
- }
- fragByte_.get
- }
-
- /**
- * Bit-oriented decoders have to be passed a standard byteBuffer and CharBuffer
- * via the normal Java decodeLoop API. However, as we are bit oriented, there's the
- * possibility that there is a fragment of a byte at the end, and there
- * can be an offset within the first byte for the start.
- *
- * So to get that information over to the decoder, given that the decodeLoop
- * API doesn't pass it, we have setters.
- *
- * So a decoder is stateful. They're always stateful, so this isn't changing
- * that. We're just adding yet more state.
- */
-
- final def setInitialBitOffset(bitOffset0to7: Int) {
- Assert.usage(!startBitOffsetHasBeenSet, "Already set. Cannot set again until decoder is reset().")
- Assert.usage(bitOffset0to7 <= 7 && bitOffset0to7 >= 0)
- startBitOffset0b = bitOffset0to7
- startBitOffsetHasBeenSet = true
- }
-
- /**
- * If set, then it means there IS one more byte (at least) at the end of
- * the ByteBuffer which holds the fragment byte. This final byte is past the
- * limit of the ByteBuffer.
- */
- final def setFinalByteBitLimitOffset0b(bitLimitOffset0b: MaybeULong) {
- maybeBitLimitOffset0b = bitLimitOffset0b
- }
-
- final protected def getFinalByteBitLimitOffset0b() = maybeBitLimitOffset0b
-
- /**
- * Calling this returns the startBitOffset0b, but also sets flags so
- * that it won't be processed again. It's a once-only thing until a reset.
- */
- final protected def takeStartBitOffset() = {
- if (startBitOffsetHasBeenUsed) 0 // one time we return the value. After that 0 until a reset.
- else {
- startBitOffsetHasBeenUsed = true
- startBitOffset0b
- }
- }
-
- final protected def resetStartBit() {
- startBitOffsetHasBeenUsed = false
- startBitOffset0b = 0
- startBitOffsetHasBeenSet = false
- }
-
- /**
- * When there is a prior byte, this gives the number of bits from it
- * that have not yet been consumed. Zero when there is no prior byte.
- *
- * Value ranges from 1 to 7.
- *
- * There is a case where it is set to 8, which is when we put
- * a new whole byte from the input into it, and then immediately
- * remove a charcode from it.
- */
- private var priorByteBitCount = 0
-
- /*
- * These predicates make reasoning about the various combinations
- * easier.
- *
- * For the input, we can have data (whole bytes), a fragment of a byte,
- * or nothing.
- *
- * For the output, we either have room for a character, nor not.
- *
- * For the decoder itself, we have whether there is a prior byte
- * of data (generally part of a byte) or not.
- */
-
- /**
- * No whole bytes are available
- */
- @inline private final def NoData(in: ByteBuffer) = !YesData(in) && NoFrag
-
- /**
- * At least 1 whole byte is available
- */
- @inline private final def YesData(in: ByteBuffer) = in.hasRemaining()
-
- /**
- * Space for at least 1 character of output
- */
- @inline private final def YesSpace(out: CharBuffer) = out.hasRemaining()
-
- /**
- * No partially consumed prior byte
- */
- private final def NoPrior = {
- val res = !YesPrior
- if (res) Assert.invariant(priorByteBitCount == 0)
- res
- }
-
- /**
- * There is a partially consumed prior byte. Contains at least 1 bit.
- *
- * When we first create a priorByte it may contain 8 bits, but that
- * is a transient condition. When this is called it should be 1 to 7
- */
- private final def YesPrior = {
- val res = priorByte.isDefined
- if (res) Assert.invariant(priorByteBitCount > 0 && priorByteBitCount <= 7)
- res
- }
-
- @inline private def takeCharCodeFromPriorByte() =
- takeBitsFromPriorByte(bitWidthOfACodeUnit)
-
- private def takeBitsFromPriorByte(nBits: Int) = {
- Assert.usage(nBits > 0)
- Assert.invariant(priorByte.isDefined)
- val cnt = priorByteBitCount
- Assert.invariant(cnt >= nBits)
- val pb = priorByte.get
- val mask = 0xFF >> (8 - nBits)
- val isolatingShift =
- if (this.requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
- cnt - nBits
- } else {
- // LSBF
- 8 - cnt
- }
- val charCode =
- (pb >>> isolatingShift) & mask
- priorByteBitCount = priorByteBitCount - nBits
- if (priorByteBitCount == 0)
- priorByte = MaybeInt.Nope
- charCode
- }
-
- @inline private def takeCharCodeFromFragByte(in: ByteBuffer): Int =
- takeBitsFromFragByte(in, bitWidthOfACodeUnit)
-
- private def takeBitsFromFragByte(in: ByteBuffer, nBits: Int): Int = {
- val fb = getFragByte(in)
- val nBitsInFrag = this.maybeBitLimitOffset0b.get
- Assert.invariant(nBitsInFrag >= nBits)
- var bits = 0
- var newFB = 0
- val mask = 0xFF >> (8 - nBits)
- if (this.requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
- // MSBF
- bits = (fb >>> (8 - nBits)) & mask
- newFB = (fb << nBits) & 0xFF
- } else {
- // LSBF
- bits = fb & mask
- newFB = (fb >>> nBits) & 0xFF
- }
- val newLimit = nBitsInFrag - nBits
- if (newLimit > 0)
- maybeBitLimitOffset0b = MaybeULong(newLimit)
- else
- maybeBitLimitOffset0b = MaybeULong.Nope
- fragByte_ =
- if (maybeBitLimitOffset0b.isEmpty)
- MaybeInt.Nope
- else
- MaybeInt(newFB)
- bits
- }
-
- /**
- * No partial fragment byte at the end of the data (after all whole bytes)
- */
- private def NoFrag = {
- val res = maybeBitLimitOffset0b.isEmpty
- if (res)
- Assert.invariant(fragByte_.isEmpty)
- res
- }
-
- /**
- * There is a fragment byte after all whole bytes. Contains at least 1 bit.
- */
- private def YesFrag(in: ByteBuffer) = {
- val res = maybeBitLimitOffset0b.isDefined && !in.hasRemaining()
- if (res) {
- val lim = maybeBitLimitOffset0b.get
- if (lim == 0) Assert.invariant(fragByte_.isEmpty)
- Assert.invariant(lim >= 0 && lim <= 7)
- }
- res
- }
-
- /**
- * Returns false if not enough bits are in the data to handle the start offset.
- */
- private def handleStartOffset(in: ByteBuffer, out: CharBuffer): Boolean = {
- val bitOffset = takeStartBitOffset()
- if (bitOffset != 0) {
- Assert.invariant(NoPrior)
- // if there's a start offset, then we're just starting.
- // we have to offset into the first byte if there is one.
- if (YesData(in)) {
- priorByte = MaybeInt(Bits.asUnsignedByte(in.get()))
- priorByteBitCount = 8
- this.takeBitsFromPriorByte(bitOffset)
- // fall through to main loop below, but now we have a prior byte, and no start offset.
- true
- } else if (YesFrag(in)) {
- // no whole bytes of data. But there could still be a frag byte
- if (this.maybeBitLimitOffset0b.get >= bitOffset) {
- // we have to have enough bits in the frag.
- // TODO: TBD should the above be a return UNDERFLOW instead of abort?
- takeBitsFromFragByte(in, bitOffset)
- true
- } else false
- } else {
- false
- }
- } else
- true
- }
-
- final def decodeLoop(in: ByteBuffer, out: CharBuffer): CoderResult = {
- init
- //
- // Now we have to adjust for the starting bit offset
- // Grab first byte so we can start at an offset into it.
- //
- if (!handleStartOffset(in, out)) {
- CoderResult.UNDERFLOW // not enough data to handle the starting offset
- } else {
- //
- // Once around this loop per character decoded.
- //
- // Note that we can enter this loop at any state. E.g., we could have pulled
- // a byte from the input, decoded some bits from it (to go with earlier bits)
- // but still have more to go in that character, but have to return from this
- // loop due to charBuf overflow. When we re-enter this loop we
- // presumably will have more charBuf space, and we have to pick up where
- // we left off.
- //
- // States:
- // YesPrior or NoPrior - byte currently being consumed
- // Output: YesSpace or NoSpace
- // Input: YesData (more whole bytes), YesFrag (partial final byte), NoData (no more data of any kind)
- //
- var rc: Maybe[CoderResult] = Maybe.Nope
- while (rc.isEmpty) {
- rc = attemptToDecodeOne(in, out)
- }
- rc.get
- }
- }
-
- /**
- * Attempt to decode a single character.
- *
- * Returns Maybe.Nope to indicate successful decode.
- * Returns One(CoderResult.UNDERFLOW) or One(CoderResult.OVERFLOW) to
- * indicate it was unable to decode.
- */
- private val underflow = Maybe(CoderResult.UNDERFLOW)
- private val overflow = Maybe(CoderResult.OVERFLOW)
-
- // package private - so we can call for unit testing. But this is not
- // part of the public interface of these decoders.
- //
- private[io] def attemptToDecodeOne(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
- if (YesPrior)
- attemptToDecodeOneYesPrior(in, out)
- else
- attemptToDecodeOneNoPrior(in, out)
- }
-
- private def attemptToDecodeOneYesPrior(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
- //
- // YesPrior Cases - where there is a prior byte, and it has enough
- // bits to produce a character just from there.
- //
- if (priorByteBitCount >= bitWidthOfACodeUnit) {
- if (YesSpace(out)) {
- // there is space in the charbuffer, produce a character
- val charCode = this.takeCharCodeFromPriorByte() // priorByteBitCount gets smaller.
- output(charCode, out)
- Maybe.Nope
- } else {
- overflow
- }
- } else //
- //
- // YesPrior Cases - but prior byte alone is not enough, but YesSpace
- //
- if (YesSpace(out)) {
- attemptToDecodeOneYesPriorYesSpace(in, out)
- } else {
- attemptToDecodeOneYesPriorNoSpace(in, out)
- }
- }
-
- private def attemptToDecodeOneYesPriorYesSpace(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
- if (YesData(in))
- attemptToDecodeOneYesPriorYesSpaceYesData(in, out)
- else if (YesFrag(in))
- attemptToDecodeOneYesPriorYesSpaceYesFrag(in, out)
- else if (NoData(in))
- attemptToDecodeOneYesPriorYesSpaceNoData(in, out)
- else
- Assert.impossible("All cases should have been exhausted")
- }
-
- private lazy val mask = 0xFF >>> (8 - bitWidthOfACodeUnit)
-
- def attemptToDecodeOneYesPriorYesSpaceYesData(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
- // we need more bits from the next byte. At least 1 bit comes from
- // priorByte. We will produce a character and go around again, because
- // a whole byte from input always provides enough bits.
- val nNeededBits = bitWidthOfACodeUnit - priorByteBitCount
- Assert.invariant(nNeededBits >= 1)
- val currentByte = Bits.asUnsignedByte(in.get())
- val nPriorBits = priorByteBitCount
- val priorBits = takeBitsFromPriorByte(priorByteBitCount)
- val nCurrentBits = (bitWidthOfACodeUnit - nPriorBits)
-
- val charCode =
- if (requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
- val priorBitsInPosition = priorBits << nCurrentBits
- val currentBits = currentByte >>> (8 - nCurrentBits)
- val cc = (priorBitsInPosition | currentBits) & mask
- cc
- } else {
- // LSBF
- val priorBitsInPosition = priorBits
- val currentBits = (currentByte << nPriorBits) & mask
- val cc = (priorBitsInPosition | currentBits) & mask
- cc
- }
- priorByte = MaybeInt(currentByte)
- priorByteBitCount = 8 - nCurrentBits // priorByte will become the unconsumed bits from data byte.
- output(charCode, out)
- Maybe.Nope
- }
-
- def attemptToDecodeOneYesPriorYesSpaceYesFrag(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
- if (priorByteBitCount + this.maybeBitLimitOffset0b.get >= bitWidthOfACodeUnit) {
- // Case where combining prior with frag gets us enough bits for a character
- // We will produce a character and go around again.
- val nPriorBits = priorByteBitCount
- val priorBits = takeBitsFromPriorByte(priorByteBitCount)
- val nCurrentBits = (bitWidthOfACodeUnit - nPriorBits)
- val currentBits = takeBitsFromFragByte(in, nCurrentBits)
- val charCode =
- if (requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
- val priorBitsInPosition = priorBits << nCurrentBits
- val cc = (priorBitsInPosition | currentBits) & mask
- cc
- } else {
- // LSBF
- val currentBitsInPosition = currentBits << nPriorBits
- val cc = (priorBits | currentBitsInPosition) & mask
- cc
- }
- priorByte = MaybeInt.Nope
- priorByteBitCount = 0
- output(charCode, out)
- // No more prior byte, and frag byte gets smaller.
- Maybe.Nope
- } else {
- // Not enough bits left period combining prior and frag.
- underflow
- }
- }
-
- def attemptToDecodeOneYesPriorYesSpaceNoData(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
- //
- // Keep in mind the invariant. There is NOT enoug bits in the prior for a character
- //
- Assert.invariant(priorByteBitCount < bitWidthOfACodeUnit)
- underflow
- }
-
- private def attemptToDecodeOneNoPrior(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
- if (YesSpace(out))
- attemptToDecodeOneNoPriorYesSpace(in, out)
- else
- attemptToDecodeOneNoPriorNoSpace(in, out)
- }
-
- private def attemptToDecodeOneNoPriorYesSpace(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
- //
- //////////////////////////////////
- // NoPrior Cases - but YesSpace
- //////////////////////////////////
- //
- if (YesData(in)) {
- //
- // Fresh start or we hit a clean byte boundary
- //
- priorByte = MaybeInt(Bits.asUnsignedByte(in.get()))
- priorByteBitCount = 8
- val currentCharCode = takeCharCodeFromPriorByte()
- output(currentCharCode, out)
- Maybe.Nope
- } else if (YesFrag(in)) {
- // There is a partial frag byte available in the data, and that may provide
- // sufficient bits to decode a character. If enough, decode one & loop
- // reducing the size of the frag.
- //
- val fragLen = maybeBitLimitOffset0b.get
- if (fragLen >= bitWidthOfACodeUnit) {
- val charCode = this.takeCharCodeFromFragByte(in)
- output(charCode, out)
- // frag gets smaller, or disappears.
- Maybe.Nope
- } else {
- // frag is insufficient to create a character
- underflow
- }
- } else if (NoData(in)) {
- underflow
- } else {
- Assert.impossible("All cases are supposed to have been exhausted.")
- }
- }
-
- private def attemptToDecodeOneNoPriorNoSpace(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
- //
- //////////////////////////////////
- // NoPrior, but NoSpace
- //
- // We'll never produce a character here
- // We always return with some CoderResult status.
- //////////////////////////////////
- if (YesData(in)) {
- overflow
- } else if (YesFrag(in)) {
- val fragLen = maybeBitLimitOffset0b.get
- if (fragLen >= bitWidthOfACodeUnit) {
- // if frag has enough for a character, then overflow.
- overflow
- } else {
- underflow
- }
- } else if (NoData(in)) {
- underflow
- } else {
- Assert.impossible("All cases are supposed to have been exhausted.")
- }
- }
-
- private def attemptToDecodeOneYesPriorNoSpace(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
- ///////////////////////////////
- // YesPrior but NoSpace Cases
- ///////////////////////////////
- if (YesData(in)) {
- overflow
- } else if (YesFrag(in)) {
- if (priorByteBitCount + this.maybeBitLimitOffset0b.get >= bitWidthOfACodeUnit) {
- overflow
- } else {
- // Not enough bits left combining prior and frag.
- // Note that we prefer underflow to overflow. There's no
- // character to produce here, so we haven't run out of space in the
- // output, we're unable to produce one.
- underflow
- }
- } else if (NoData(in)) {
- underflow
- } else {
- Assert.impossible("All cases are supposed to have been exhausted.")
- }
- }
-
- def output(charCode: Int, out: CharBuffer)
-}
-
-trait NonByteSizeCharsetEncoder
- extends NonByteSizeCharset { self: java.nio.charset.CharsetEncoder =>
-
- def replacementChar: Int
-
- private var partialByte: Int = 0
- private var partialByteLenInBits: Int = 0
-
- protected override def implReset() {
- partialByte = 0
- partialByteLenInBits = 0
- }
-
- def encodeLoop(in: CharBuffer, out: ByteBuffer): CoderResult = {
- init
- while (true) {
- val inHasRemainingData = in.hasRemaining()
- val outHasRemainingSpace = out.hasRemaining()
-
- if (!inHasRemainingData) {
- if (partialByteLenInBits > 0) {
- if (!outHasRemainingSpace) {
- // no remaining input, but there's a partial byte we need to write,
- // and nowhere to write it to
- return CoderResult.OVERFLOW
- } else {
- // no remaining input, but there's a partial byte, write the partial
- // byte (includes padding) and finish
- out.put(partialByte.toByte)
- partialByte = 0
- partialByteLenInBits = 0
- return CoderResult.UNDERFLOW
- }
- } else {
- // no remaining input, and no partial byte, nothing left to
- // encode/write, finish
- return CoderResult.UNDERFLOW
- }
- } else {
- if (!outHasRemainingSpace) {
- // there's input to encode, but nowhere to write it to
- return CoderResult.OVERFLOW
- }
-
- // there's data to encode and enough space to write a byte, encode the
- // character
-
- val charCode = charToCharCode(in.get())
-
- val charCodeToWrite =
- if (charCode.isDefined) {
- charCode.get
- } else {
- // character must fit in the bit width of a code unit, unmappable error
- val unmappableAction = self.unmappableCharacterAction
- if (unmappableAction == CodingErrorAction.REPLACE) {
- // CharsetEncoder, which handles character replacement, assumes
- // that the replacement character is made up of full bytes. That
- // isn't the case for this charset, so we will just manually
- // replace the character ourselves and not let CharsetEncoder
- // know there was ever a problem.
- replacementChar
- } else {
- Assert.invariant(unmappableAction == CodingErrorAction.REPORT)
- // Just report the unmappable character. Note that we should back
- // up the position of the CharBuffer since we read a character
- // but it was invalid. The charset encoder that called encodeLoop
- // will handle skipping the invalid character. Note that the
- // character is only skipped in CodingErrorAction.IGNORE and
- // REPLACE. We handle REPLACE ourselves, and IGNORE isn't
- // supported by DFDL, so this *shouldn't* matter, but it might
- // someday if IGNORE is supported.
- in.position(in.position() - 1)
- return CoderResult.unmappableForLength(1)
- }
- }
-
- if (partialByteLenInBits == 0) {
- // no partial byte exists, make this the partial byte
- if (requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
- partialByte = charCodeToWrite << (8 - bitWidthOfACodeUnit)
- } else {
- partialByte = charCodeToWrite
- }
- partialByteLenInBits = bitWidthOfACodeUnit
- } else if ((partialByteLenInBits + bitWidthOfACodeUnit) >= 8) {
- // This character's bits will fill up the byte. Some bits might be left over.
- // There's a partial byte, add enough bits to make it a full byte and
- // write it, then save whatever is remaining (could be 0 bits
- // remaining) as a partial byte
- val nUsedBits = 8 - partialByteLenInBits
- if (requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
- val nLeftOverBits = bitWidthOfACodeUnit - nUsedBits
- Assert.invariant(nLeftOverBits >= 0)
- partialByte |= (charCodeToWrite >> nLeftOverBits) & 0xFF
- out.put(partialByte.toByte)
- val leftOverMask = (1 << nLeftOverBits) - 1
- val newPartialByte = (charCodeToWrite & leftOverMask) << (8 - nLeftOverBits)
- partialByte = newPartialByte & 0xFF
- partialByteLenInBits = nLeftOverBits
- } else {
- // LSBF
- partialByte |= (charCodeToWrite << partialByteLenInBits) & 0xFF
- out.put(partialByte.toByte)
- partialByte = charCodeToWrite >> nUsedBits
- partialByteLenInBits = bitWidthOfACodeUnit - nUsedBits
- }
- } else {
- // there's a partial byte but there won't be enough bits to make it a full byte
- if (requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
- partialByte |= (charCodeToWrite << (8 - partialByteLenInBits - bitWidthOfACodeUnit)) & 0xFF
- } else {
- // LSBF
- partialByte |= (charCodeToWrite << partialByteLenInBits) & 0xFF
- }
- partialByteLenInBits = bitWidthOfACodeUnit + partialByteLenInBits
- }
-
- //
- // Verify invariant that the unused bits of the partialByte are always 0
- //
- Assert.invariant({
- val unusedPartialByteMask: Int =
- if (requiredBitOrder eq BitOrder.MostSignificantBitFirst)
- (1 << (8 - partialByteLenInBits)) - 1
- else
- (-1 << partialByteLenInBits) & 0xFF
- val unusedPartialByteBits = partialByte & unusedPartialByteMask
- unusedPartialByteBits == 0
- })
- }
- }
-
- Assert.impossible("Incorrect return from encodeLoop")
- }
-
- protected def charToCharCode(char: Char): MaybeInt
-}
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/StringDataInputStreamForUnparse.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/StringDataInputStreamForUnparse.scala
index 727daee..003505e 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/StringDataInputStreamForUnparse.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/io/StringDataInputStreamForUnparse.scala
@@ -35,6 +35,7 @@ package edu.illinois.ncsa.daffodil.io
import edu.illinois.ncsa.daffodil.util.Misc
import edu.illinois.ncsa.daffodil.exceptions.Assert
import edu.illinois.ncsa.daffodil.util.MaybeULong
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharsetWrappingJavaCharset
/**
* When unparsing, we reuse all the DFA logic to identify delimiters within
@@ -52,7 +53,7 @@ final class StringDataInputStreamForUnparse
def reset(str: String, finfo: FormatInfo) {
this.str = str
- val ba = str.getBytes(finfo.decoder.charset())
+ val ba = str.getBytes(finfo.decoder.bitsCharset.asInstanceOf[BitsCharsetWrappingJavaCharset].javaCharset)
dis = ByteBufferDataInputStream(ba)
}
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/BitsCharset.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/BitsCharset.scala
new file mode 100644
index 0000000..bacf5e6
--- /dev/null
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/BitsCharset.scala
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.illinois.ncsa.daffodil.processors.charset
+
+import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
+import edu.illinois.ncsa.daffodil.exceptions.Assert
+import java.nio.charset.CoderResult
+import java.nio.charset.CodingErrorAction
+import java.nio.charset.{ CharsetEncoder => JavaCharsetEncoder }
+import java.nio.charset.{ CharsetDecoder => JavaCharsetDecoder }
+import java.nio.charset.{ StandardCharsets => JavaStandardCharsets }
+import java.nio.charset.{ Charset => JavaCharset }
+import java.nio.CharBuffer
+import java.nio.ByteBuffer
+import edu.illinois.ncsa.daffodil.util.MaybeULong
+import edu.illinois.ncsa.daffodil.util.MaybeInt
+
+/**
+ * Charset enhanced with features allowing it to work with Daffodil's Bit-wise
+ * DataInputStream and DataOutputStream.
+ *
+ * Daffodil uses BitsCharset as its primary abstraction for dealing with
+ * character sets, which enables it to support character sets where the code
+ * units are smaller than 1 byte.
+ *
+ * Note that BitsCharset is NOT derived from java.nio.charset.Charset, nor
+ * are BitsCharsetDecoder or BitsCharsetEncoder derived from
+ * java.nio.charset.CharsetDecoder or CharsetEncoder respectively. This is because
+ * these Java classes have many final methods that make it impossible for us
+ * to implement what we need by extending them. Hence, a BitsCharset is its own
+ * interface that has some similarities to the Java charset-related classes.
+ *
+ * For regular byte-centric charsets a BitsCharset is layered on top of
+ * Java's java.nio.charset.Charset.
+ */
+trait BitsCharset extends Serializable {
+ final override def hashCode = name.hashCode
+ final override def equals(other: Any) = other match {
+ case bcs: BitsCharset => this.name == bcs.name
+ case _ => false
+ }
+ def name: String
+ def bitWidthOfACodeUnit: Int // in units of bits
+ def requiredBitOrder: BitOrder
+ def mandatoryBitAlignment: Int
+ def newDecoder(): BitsCharsetDecoder
+ def newEncoder(): BitsCharsetEncoder
+
+ def maybeFixedWidth: MaybeInt
+
+ final def padCharWidthInBits = {
+ if (maybeFixedWidth.isDefined)
+ maybeFixedWidth.get
+ else {
+ this match {
+ case StandardBitsCharsets.UTF_8 => 8
+ case _ => Assert.invariantFailed("Getting pad char width for unsupported variable-width charset: " + name)
+ }
+ }
+ }
+}
+
+trait IsResetMixin {
+ private final var isReset_ : Boolean = true
+ /**
+ * True if the decoder has not decoded anything since the last reset call.
+ * False if decodeLoop has been called.
+ *
+ * Use to control things that we only want to check once per reset of the
+ * decoder.
+ */
+ final def isReset = isReset_
+ /**
+ * Allow assignment to isReset only in derived classes
+ */
+ protected final def isReset_=(v: Boolean): Unit = isReset_ = v
+}
+
+trait BitsCharsetDecoder
+ extends IsResetMixin {
+ def bitsCharset: BitsCharset
+ def setInitialBitOffset(offset: Int): Unit
+ def setFinalByteBitLimitOffset0b(bitLimitOffset0b: MaybeULong): Unit
+ def averageCharsPerByte(): Float
+ def maxCharsPerByte(): Float
+ def averageCharsPerBit(): Float
+ def maxCharsPerBit(): Float
+ def replacement(): String
+ def replaceWith(newReplacement: String): BitsCharsetDecoder
+ def flush(out: CharBuffer): CoderResult
+ def reset(): BitsCharsetDecoder
+
+ /**
+ * Used to determine if the data input stream must be aligned (if not already)
+ * for this encoding. Based on whether the coder has been reset. If the
+ * coder has not been reset, it is assumed we are in the middle of decoding
+ * many characters, and so no mandatory alignment is needed. However, if the
+ * coder was reset, then it is assumed that we may be unaligned at the start
+ * of decoding characters, and so we must check if we are mandatory aligned.
+ */
+ def isMandatoryAlignmentNeeded(): Boolean
+ def malformedInputAction(): CodingErrorAction
+ def onMalformedInput(action: CodingErrorAction): BitsCharsetDecoder
+ def unmappableCharacterAction(): CodingErrorAction
+ def onUnmappableCharacter(action: CodingErrorAction): BitsCharsetDecoder
+ def decode(in: ByteBuffer, out: CharBuffer, endOfInput: Boolean): CoderResult
+ protected def decodeLoop(in: ByteBuffer, out: CharBuffer): CoderResult
+
+ final def decode(in: ByteBuffer): CharBuffer = {
+ var n = scala.math.ceil(in.remaining() * averageCharsPerByte()).toInt
+ var out = CharBuffer.allocate(n)
+
+ if ((n == 0) && (in.remaining() == 0)) out
+ else {
+ reset()
+ var break = false
+ while (!break) {
+ var cr =
+ if (in.hasRemaining())
+ decode(in, out, true)
+ else
+ CoderResult.UNDERFLOW
+ if (cr.isUnderflow())
+ cr = flush(out)
+ if (cr.isUnderflow())
+ break = true
+ else if (cr.isOverflow()) {
+ n = 2 * n + 1; // Ensure progress; n might be 0!
+ val o = CharBuffer.allocate(n)
+ out.flip()
+ o.put(out)
+ out = o
+ } else
+ cr.throwException()
+ }
+ out.flip()
+ out
+ }
+ }
+}
+
+abstract class BitsCharsetEncoder
+ extends IsResetMixin {
+ def bitsCharset: BitsCharset
+ def averageBytesPerChar(): Float
+ def maxBytesPerChar(): Float
+ def averageBitsPerChar(): Float
+ def maxBitsPerChar(): Float
+ def replacement(): Array[Byte]
+ def replaceWith(newReplacement: Array[Byte]): BitsCharsetEncoder
+ def flush(out: ByteBuffer): CoderResult
+ def reset(): BitsCharsetEncoder
+
+ /**
+ * Used to determine if the data input stream must be aligned (if not already)
+ * for this encoding. Based on whether the coder has been reset. If the
+ * coder has not been reset, it is assumed we are in the middle of encoding
+ * many characters, and so no mandatory alignment is needed. However, if the
+ * coder was reset, then it is assumed that we may be unaligned at the start
+ * of encoding characters, and so we must check if we are mandatory aligned.
+ */
+ def isMandatoryAlignmentNeeded(): Boolean
+ def malformedInputAction(): CodingErrorAction
+ def onMalformedInput(action: CodingErrorAction): BitsCharsetEncoder
+ def unmappableCharacterAction(): CodingErrorAction
+ def onUnmappableCharacter(action: CodingErrorAction): BitsCharsetEncoder
+ def encode(in: CharBuffer, out: ByteBuffer, endOfInput: Boolean): CoderResult
+ protected def encodeLoop(in: CharBuffer, out: ByteBuffer): CoderResult
+}
+
+/**
+ * Implements BitsCharset based on encapsulation of a regular JavaCharset.
+ */
+final class BitsCharsetWrappingJavaCharset(nameArg: String)
+ extends BitsCharset {
+
+ javaCharset // Force javaCharset to be evaluated to ensure it's valid at compile time.
+ // It's a lazy val so it will be evaluated when de-serialized
+ @transient lazy val name = javaCharset.name
+ @transient lazy val javaCharset = JavaCharset.forName(nameArg)
+ @transient final override lazy val maybeFixedWidth = CharsetUtils.maybeEncodingFixedWidth(this)
+
+ override def newDecoder() = new BitsCharsetWrappingJavaCharsetDecoder(this, javaCharset.newDecoder())
+ override def newEncoder() = new BitsCharsetWrappingJavaCharsetEncoder(this, javaCharset.newEncoder())
+
+ override def bitWidthOfACodeUnit: Int = 8
+ override def requiredBitOrder = BitOrder.MostSignificantBitFirst // really none, as these are mandatory aligned to byte boundary.
+ override def mandatoryBitAlignment = 8
+
+}
+
+/**
+ * Implements BitsCharsetDecoder by encapsulation of a standard
+ * JavaCharsetDecoder.
+ */
+final class BitsCharsetWrappingJavaCharsetDecoder(override val bitsCharset: BitsCharsetWrappingJavaCharset, dec: JavaCharsetDecoder)
+ extends BitsCharsetDecoder {
+
+ def setInitialBitOffset(offset: Int): Unit = Assert.usageError("Not to be called.")
+ def setFinalByteBitLimitOffset0b(bitLimitOffset0b: MaybeULong): Unit = Assert.usageError("Not to be called.")
+
+ def averageCharsPerByte() = dec.averageCharsPerByte()
+ def averageCharsPerBit() = averageCharsPerByte() / 8.0F
+ def maxCharsPerByte() = dec.maxCharsPerByte()
+ def maxCharsPerBit() = maxCharsPerByte() / 8.0F
+ def replacement() = dec.replacement()
+ def replaceWith(newReplacement: String) = {
+ Assert.usage(isReset)
+ dec.replaceWith(newReplacement); this
+ }
+ def flush(out: CharBuffer) = {
+ Assert.usage(!isReset)
+ dec.flush(out)
+ }
+ def reset() = {
+ dec.reset();
+ isReset = true
+ this
+ }
+ def isMandatoryAlignmentNeeded() = isReset
+
+ def malformedInputAction() = dec.malformedInputAction()
+ def onMalformedInput(action: CodingErrorAction) = {
+ Assert.usage(isReset)
+ dec.onMalformedInput(action);
+ this
+ }
+ def unmappableCharacterAction() = dec.malformedInputAction()
+ def onUnmappableCharacter(action: CodingErrorAction) = {
+ Assert.usage(isReset)
+ dec.onUnmappableCharacter(action);
+ this
+ }
+ def decode(in: ByteBuffer, out: CharBuffer, endOfInput: Boolean) = {
+ isReset = false
+ dec.decode(in, out, endOfInput)
+ }
+ protected def decodeLoop(in: ByteBuffer, out: CharBuffer): CoderResult =
+ Assert.usageError("Not to be called.")
+
+}
+
+/**
+ * Implements BitsCharsetEncoder by encapsulating a standard JavaCharsetEncoder
+ */
+final class BitsCharsetWrappingJavaCharsetEncoder(override val bitsCharset: BitsCharsetWrappingJavaCharset, enc: JavaCharsetEncoder)
+ extends BitsCharsetEncoder {
+
+ def setInitialBitOffset(offset: Int): Unit = Assert.usageError("Not to be called.")
+ def averageBytesPerChar() = enc.averageBytesPerChar()
+ def averageBitsPerChar() = averageBytesPerChar() / 8.0F
+ def maxBytesPerChar() = enc.maxBytesPerChar()
+ def maxBitsPerChar() = maxBytesPerChar() / 8.0F
+ def replacement() = enc.replacement()
+ def replaceWith(newReplacement: Array[Byte]) = {
+ Assert.usage(isReset)
+ enc.replaceWith(newReplacement);
+ this
+ }
+ def flush(out: ByteBuffer) = {
+ Assert.usage(!isReset)
+ enc.flush(out)
+ }
+ def reset() = {
+ enc.reset();
+ isReset = true
+ this
+ }
+ def isMandatoryAlignmentNeeded = isReset
+ def malformedInputAction() = enc.malformedInputAction()
+ def onMalformedInput(action: CodingErrorAction) = {
+ Assert.usage(isReset)
+ enc.onMalformedInput(action);
+ this
+ }
+ def unmappableCharacterAction() = enc.malformedInputAction()
+ def onUnmappableCharacter(action: CodingErrorAction) = {
+ Assert.usage(isReset)
+ enc.onUnmappableCharacter(action);
+ this
+ }
+ def encode(in: CharBuffer, out: ByteBuffer, endOfInput: Boolean) = {
+ isReset = false
+ enc.encode(in, out, endOfInput)
+ }
+ protected def encodeLoop(in: CharBuffer, out: ByteBuffer) =
+ Assert.usageError("Not to be called")
+}
+
+/**
+ * Provides BitsCharset objects corresponding to the usual java charsets found
+ * in StandardCharsets.
+ */
+object StandardBitsCharsets {
+ val UTF_8 = new BitsCharsetWrappingJavaCharset(JavaStandardCharsets.UTF_8.name)
+ val UTF_16BE = new BitsCharsetWrappingJavaCharset(JavaStandardCharsets.UTF_16BE.name)
+ val UTF_16LE = new BitsCharsetWrappingJavaCharset(JavaStandardCharsets.UTF_16LE.name)
+ val ISO_8859_1 = new BitsCharsetWrappingJavaCharset(JavaStandardCharsets.ISO_8859_1.name)
+}
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/CharsetUtils.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/CharsetUtils.scala
index 8016940..e9fe5fc 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/CharsetUtils.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/CharsetUtils.scala
@@ -34,55 +34,24 @@ package edu.illinois.ncsa.daffodil.processors.charset
import java.nio.ByteBuffer
import java.nio.CharBuffer
-import java.nio.charset.Charset
-import java.nio.charset.CharsetDecoder
-import java.nio.charset.CharsetEncoder
+import java.nio.charset.{ Charset => JavaCharset }
import java.nio.charset.CoderResult
import java.nio.charset.CodingErrorAction
import edu.illinois.ncsa.daffodil.exceptions.Assert
import edu.illinois.ncsa.daffodil.io.LocalBufferMixin
-import edu.illinois.ncsa.daffodil.io.NonByteSizeCharset
import edu.illinois.ncsa.daffodil.util.MaybeInt
-/*
- * These are needed because the ordinary java/scala Charset, Encoder, and Decoder objects
- * are not serializable. So we must go back to the string from which they can be created
- * in order to serialize these.
- */
-
-/**
- * Serializable Charset
- */
-case class DFDLCharset(val charsetName: String) extends Serializable {
- import java.nio.charset.StandardCharsets
-
- charset // Force charset to be evaluted to ensure it's valid at compile time. It's a lazy val so it will be evaluated when de-serialized
- @transient lazy val charset = CharsetUtils.getCharset(charsetName)
- @transient lazy val maybeFixedWidth = CharsetUtils.maybeEncodingFixedWidth(charset)
-
- def padCharWidthInBits = {
- if (maybeFixedWidth.isDefined)
- maybeFixedWidth.get
- else {
- charset match {
- case StandardCharsets.UTF_8 => 8
- case _ => Assert.invariantFailed("unsupported charset: " + charset)
- }
- }
- }
-}
-
object CharsetUtils {
/**
* Call instead of Charset.forName to obtain Daffodil's less-than-byte-sized
* encodings as well as the standard ones.
*/
- def getCharset(name: String) = {
+ def getCharset(name: String): BitsCharset = {
val dcs = DaffodilCharsetProvider.charsetForName(name)
val cs =
if (dcs ne null) dcs
- else Charset.forName(name)
+ else new BitsCharsetWrappingJavaCharset(name)
cs
}
@@ -95,7 +64,7 @@ object CharsetUtils {
* Java 7 at some point in the future.
*/
lazy val hasJava7DecoderBug = {
- val decoder = Charset.forName("utf-8").newDecoder()
+ val decoder = JavaCharset.forName("utf-8").newDecoder()
decoder.onMalformedInput(CodingErrorAction.REPORT)
decoder.onUnmappableCharacter(CodingErrorAction.REPORT)
val bb = ByteBuffer.allocate(6)
@@ -121,9 +90,9 @@ object CharsetUtils {
* Tells us the encoding's fixed width if it is fixed.
* Nope if not fixed width
*/
- final def maybeEncodingFixedWidth(charset: Charset): MaybeInt = {
+ final def maybeEncodingFixedWidth(charset: BitsCharset): MaybeInt = {
val res: Int = charset match {
- case nbs: NonByteSizeCharset => nbs.bitWidthOfACodeUnit
+ case nbs: NBitsWidth_BitsCharset => nbs.bitWidthOfACodeUnit
case _ => {
val enc = charset.newEncoder()
val avg = enc.averageBytesPerChar()
@@ -140,11 +109,11 @@ object CharsetUtils {
sealed abstract class CoderInfo(val encodingMandatoryAlignmentInBits: Int, val maybeCharWidthInBits: MaybeInt)
-case class DecoderInfo(coder: CharsetDecoder, replacingCoder: CharsetDecoder, reportingCoder: CharsetDecoder,
+case class DecoderInfo(coder: BitsCharsetDecoder, replacingCoder: BitsCharsetDecoder, reportingCoder: BitsCharsetDecoder,
encodingMandatoryAlignmentInBitsArg: Int, maybeCharWidthInBitsArg: MaybeInt)
extends CoderInfo(encodingMandatoryAlignmentInBitsArg, maybeCharWidthInBitsArg)
-case class EncoderInfo(coder: CharsetEncoder, replacingCoder: CharsetEncoder, reportingCoder: CharsetEncoder,
+case class EncoderInfo(coder: BitsCharsetEncoder, replacingCoder: BitsCharsetEncoder, reportingCoder: BitsCharsetEncoder,
encodingMandatoryAlignmentInBitsArg: Int, maybeCharWidthInBitsArg: MaybeInt)
extends CoderInfo(encodingMandatoryAlignmentInBitsArg, maybeCharWidthInBitsArg)
@@ -161,12 +130,12 @@ trait EncoderDecoderMixin
* except when adding a new not-seen-before encoder or decoder.
*/
- private lazy val decoderCache = new java.util.HashMap[Charset, DecoderInfo]
- private lazy val encoderCache = new java.util.HashMap[Charset, EncoderInfo]
+ private lazy val decoderCache = new java.util.HashMap[BitsCharset, DecoderInfo]
+ private lazy val encoderCache = new java.util.HashMap[BitsCharset, EncoderInfo]
- private def derivations(charset: Charset) = {
+ private def derivations(charset: BitsCharset) = {
val tuple = charset match {
- case nbsc: NonByteSizeCharset => {
+ case nbsc: NBitsWidth_BitsCharset => {
val encodingMandatoryAlignmentInBits = 1
val maybeCharWidthInBits = MaybeInt(nbsc.bitWidthOfACodeUnit)
(encodingMandatoryAlignmentInBits, maybeCharWidthInBits)
@@ -187,8 +156,8 @@ trait EncoderDecoderMixin
tuple
}
- def getDecoder(charset: Charset) = getDecoderInfo(charset).coder
- def getDecoderInfo(charset: Charset) = {
+ def getDecoder(charset: BitsCharset) = getDecoderInfo(charset).coder
+ def getDecoderInfo(charset: BitsCharset) = {
// threadCheck()
var entry = decoderCache.get(charset)
if (entry eq null) {
@@ -206,8 +175,8 @@ trait EncoderDecoderMixin
entry
}
- def getEncoder(charset: Charset) = getEncoderInfo(charset).coder
- def getEncoderInfo(charset: Charset) = {
+ def getEncoder(charset: BitsCharset) = getEncoderInfo(charset).coder
+ def getEncoderInfo(charset: BitsCharset) = {
// threadCheck()
var entry = encoderCache.get(charset)
if (entry eq null) {
@@ -231,9 +200,9 @@ trait EncoderDecoderMixin
* This only has to scan the string if the charset is a variable-width
* encoding. Otherwise this is just constant time.
*/
- final def lengthInBits(str: String, dcharset: DFDLCharset): Long = {
+ final def lengthInBits(str: String, bitsCharset: BitsCharset): Long = {
if (str.length == 0) return 0
- val mew = dcharset.maybeFixedWidth
+ val mew = bitsCharset.maybeFixedWidth
if (mew.isDefined) {
val w = mew.get
w * str.length
@@ -255,14 +224,12 @@ trait EncoderDecoderMixin
// replaced by something. And just take the resulting length.
//
// in a mode where the errors are ignored
- val cs = dcharset.charset
-
withLocalByteBuffer { lbb =>
withLocalCharBuffer { cbb =>
val cb = cbb.getBuf(str.length)
cb.append(str)
cb.flip()
- val encoder = getEncoder(cs)
+ val encoder = getEncoder(bitsCharset)
// it is assumed here that the encoder's behavior with respect to
// dfdl:encodingErrorPolicy is already set up.
@@ -297,11 +264,11 @@ trait EncoderDecoderMixin
*
* That unused space is supposed to get filled by the RightFillUnparser with fillbyte
*/
- final def truncateToBits(str: String, dcharset: DFDLCharset, nBits: Long): String = {
+ final def truncateToBits(str: String, bitsCharset: BitsCharset, nBits: Long): String = {
Assert.usage(nBits >= 0)
if (str.length == 0) return str
if (nBits == 0) return ""
- val mew = dcharset.maybeFixedWidth
+ val mew = bitsCharset.maybeFixedWidth
if (mew.isDefined) {
val w = mew.get
val nChars = nBits / w
@@ -313,8 +280,6 @@ trait EncoderDecoderMixin
// charset is variable width encoding
//
// We want to get a byteBuffer
- val cs = dcharset.charset
-
withLocalByteBuffer { lbb =>
val nBytes = nBits / 8
val bb = lbb.getBuf(nBytes)
@@ -322,7 +287,7 @@ trait EncoderDecoderMixin
val cb = cbb.getBuf(str.length)
cb.append(str)
cb.flip()
- val encoder = getEncoder(cs)
+ val encoder = getEncoder(bitsCharset)
//
// it is assumed here that the encoder's behavior with respect to
// dfdl:encodingErrorPolicy is already set up.
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/DFDL5BitPackedDecoder.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/DFDL5BitPackedDecoder.scala
index 2102667..503090a 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/DFDL5BitPackedDecoder.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/DFDL5BitPackedDecoder.scala
@@ -44,7 +44,7 @@ import edu.illinois.ncsa.daffodil.util.MaybeInt
*/
object DFDL5BitPackedLSBFCharset
- extends NBitsWidthCharset("X-DFDL-5-BIT-PACKED-LSBF",
+ extends NBitsWidth_BitsCharset("X-DFDL-5-BIT-PACKED-LSBF",
"01234567ABCDEFGHJKLMNPQRSTUVWXYZ",
5, // width
BitOrder.LeastSignificantBitFirst,
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/DaffodilCharsetProvider.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/DaffodilCharsetProvider.scala
index ca1ead7..8324375 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/DaffodilCharsetProvider.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/DaffodilCharsetProvider.scala
@@ -32,9 +32,7 @@
package edu.illinois.ncsa.daffodil.processors.charset
-import java.nio.charset.Charset
import edu.illinois.ncsa.daffodil.exceptions.Assert
-import java.nio.charset.spi.CharsetProvider
/**
* This implements a CharsetProvider that implements Daffodil's support
@@ -43,10 +41,14 @@ import java.nio.charset.spi.CharsetProvider
* extension mechanism was really intended for JVM maintainers to use, but
* not for end-user jar libraries to use.
*
+ * Also, we have to have a proxy class BitsCharset, and we have to produce
+ * those for all charsets, rather than being able to extend
+ * java.nio.charset.Charset - because that's full of final methods.
+ *
* Rather than contort to try to get this working, We simply call this
* directly from a CharsetUtils method that we use instead of Charset.forName.
*/
-object DaffodilCharsetProvider extends CharsetProvider {
+object DaffodilCharsetProvider {
import collection.JavaConverters._
/**
@@ -56,7 +58,7 @@ object DaffodilCharsetProvider extends CharsetProvider {
*
* @return The new iterator
*/
- override def charsets(): java.util.Iterator[Charset] = {
+ def charsets(): java.util.Iterator[BitsCharset] = {
addedCharsets.toIterator.asJava
}
@@ -65,7 +67,7 @@ object DaffodilCharsetProvider extends CharsetProvider {
private lazy val addedCharsets = addedCharsetsMap.map { _._2 }
- private val addedCharsetsMap: Map[String, Charset] =
+ private val addedCharsetsMap: Map[String, BitsCharset] =
Map(
"X-DFDL-US-ASCII-7-BIT-PACKED" -> USASCII7BitPackedCharset, // DFDL v1.0
"X-DFDL-US-ASCII-6-BIT-PACKED" -> USASCII6BitPackedCharset, // DFDL v1.0
@@ -93,7 +95,7 @@ object DaffodilCharsetProvider extends CharsetProvider {
* or <tt>null</tt> if the named charset
* is not supported by this provider
*/
- override def charsetForName(charsetName: String): Charset = {
+ def charsetForName(charsetName: String): BitsCharset = {
Assert.usage(charsetName != null);
// {
// // TODO: expensive check, remove if unnecessary
@@ -104,7 +106,7 @@ object DaffodilCharsetProvider extends CharsetProvider {
// Assert.usage(m.matches)
// }
val lookupResult = addedCharsetsMap.get(charsetName)
- val cs: Charset = lookupResult.getOrElse(null)
+ val cs = lookupResult.getOrElse(null)
cs
}
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/HexLSBF4BitDecoder.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/HexLSBF4BitDecoder.scala
index 7a6b951..f41b0fa 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/HexLSBF4BitDecoder.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/HexLSBF4BitDecoder.scala
@@ -43,14 +43,14 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
*/
object HexLSBF4BitCharset
- extends NBitsWidthCharset("X-DFDL-HEX-LSBF",
+ extends NBitsWidth_BitsCharset("X-DFDL-HEX-LSBF",
"0123456789ABCDEF",
4, // width
BitOrder.LeastSignificantBitFirst,
0) // replacement charCode for encoding of unmapped chars.
object HexMSBF4BitCharset
- extends NBitsWidthCharset("X-DFDL-HEX-MSBF",
+ extends NBitsWidth_BitsCharset("X-DFDL-HEX-MSBF",
"0123456789ABCDEF",
4, // width
BitOrder.MostSignificantBitFirst,
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/NBitsWidthCharsetBase.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/NBitsWidthCharsetBase.scala
index 4822e95..3344164 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/NBitsWidthCharsetBase.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/NBitsWidthCharsetBase.scala
@@ -32,41 +32,79 @@
package edu.illinois.ncsa.daffodil.processors.charset
-import java.nio.CharBuffer
-import java.nio.charset.Charset
-import java.nio.charset.CharsetDecoder
-import java.nio.charset.CharsetEncoder
+import edu.illinois.ncsa.daffodil.exceptions.Assert
+import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
+import edu.illinois.ncsa.daffodil.util.MaybeInt
+import edu.illinois.ncsa.daffodil.util.MaybeULong
import edu.illinois.ncsa.daffodil.exceptions.Assert
-import edu.illinois.ncsa.daffodil.io.NonByteSizeCharset
-import edu.illinois.ncsa.daffodil.io.NonByteSizeCharsetDecoder
-import edu.illinois.ncsa.daffodil.io.NonByteSizeCharsetEncoder
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
+import java.nio.ByteBuffer
+import java.nio.CharBuffer
+import edu.illinois.ncsa.daffodil.util.Bits
+import java.nio.charset.CoderResult
+import java.nio.charset.CodingErrorAction
import edu.illinois.ncsa.daffodil.util.MaybeInt
+import edu.illinois.ncsa.daffodil.util.Maybe
+
+import java.nio.charset.{ CharsetDecoder => JavaCharsetDecoder }
+import java.nio.charset.{ CharsetEncoder => JavaCharsetEncoder }
+import java.nio.charset.{ Charset => JavaCharset }
/**
* Some encodings are not byte-oriented.
*
* If we know the correspondence from integers to characters, and we can express that
* as a string, then everything else can be derived
+ *
+ * This class is explicitly not a java.nio.charset.Charset.
+ * It is a BitsCharset, which is not a compatible type with a java.nio.charset.Charset
+ * on purpose so we don't confuse the two.
+ *
+ * The problem is that java.nio.charset.Charset is designed in such a way that
+ * one cannot implement a proxy class that redirects methods to another class.
+ * This is due to all the final methods on the class.
+ *
+ * So instead we do the opposite. We implement our own BitsCharset API,
+ * but implement the behavior in terms of a proxy JavaCharsetDecoder and proxy
+ * JavaCharsetEncoder that drive the decodeLoop and encodeLoop. This way
+ * we don't have to re-implement all the error handling and flush/end logic.
*/
-class NBitsWidthCharset(name: String, decodeString: String,
+class NBitsWidth_BitsCharset(override val name: String, decodeString: String,
override val bitWidthOfACodeUnit: Int,
override val requiredBitOrder: BitOrder,
- replacementCharCode: Int)
- extends java.nio.charset.Charset(name, Array())
- with NonByteSizeCharset { self =>
+ val replacementCharCode: Int)
+ extends BitsCharset {
+
+ def averageCharsPerByte() = averageCharsPerBit() * 8.0F
+ def averageCharsPerBit() = 1.0F / bitWidthOfACodeUnit
+ def maxCharsPerByte() = maxCharsPerBit() * 8.0F
+ def maxCharsPerBit() = 1.0F / bitWidthOfACodeUnit
+ def averageBytesPerChar() = 1 / averageCharsPerByte()
+ def averageBitsPerChar() = 1 / averageCharsPerBit()
+ def maxBytesPerChar() = 1 / maxCharsPerByte()
+ def maxBitsPerChar() = 1 / maxCharsPerBit()
+
+ protected def checks() = {}
+
+ protected final lazy val init = {
+ checks()
+ }
Assert.usage(decodeString.length == math.pow(2, bitWidthOfACodeUnit))
Assert.usage(bitWidthOfACodeUnit <= 7 && bitWidthOfACodeUnit >= 1)
- def contains(cs: Charset): Boolean = false
+ Assert.usage(decodeString.length == math.pow(2, bitWidthOfACodeUnit))
+ Assert.usage(bitWidthOfACodeUnit <= 7 && bitWidthOfACodeUnit >= 1)
+
+ override def newDecoder(): BitsCharsetDecoder = new NBitsWidth_BitsCharsetDecoder(this)
- def newDecoder(): CharsetDecoder = new NBitsWidthCharsetDecoder(this)
+ override def newEncoder(): BitsCharsetEncoder = new NBitsWidth_BitsCharsetEncoder(this, replacementCharCode)
- def newEncoder(): CharsetEncoder = new NBitsWidthCharsetEncoder(this,
- replacementCharCode)
+ override def mandatoryBitAlignment = 1
+
+ override def maybeFixedWidth = MaybeInt(bitWidthOfACodeUnit)
//
// Note: because this array is only 256, these non-bit width encodings
@@ -109,41 +147,768 @@ class NBitsWidthCharset(name: String, decodeString: String,
else MaybeInt.Nope
}
-sealed trait NBitsWidthEncoderDecoderMixin {
+/**
+ * Things common to both the BitsEncoder and BitsDecoder implementations.
+ */
+sealed trait NBitsWidth_BitsEncoderDecoderMixin {
+
+ protected def cs: NBitsWidth_BitsCharset
+ final def bitWidthOfACodeUnit: Int = cs.bitWidthOfACodeUnit
+ final def requiredBitOrder: BitOrder = cs.requiredBitOrder
+
+ def averageCharsPerBit(): Float = cs.averageCharsPerBit()
+ def averageCharsPerByte(): Float = cs.averageCharsPerByte()
+ def maxCharsPerBit(): Float = cs.maxCharsPerBit()
+ def maxCharsPerByte(): Float = cs.maxCharsPerByte()
+ def bitsCharset = cs
+ def isMandatoryAlignmentNeeded(): Boolean = false
+
+ def averageBitsPerChar(): Float = cs.averageBitsPerChar()
+ def averageBytesPerChar(): Float = cs.averageBytesPerChar()
+ def maxBitsPerChar(): Float = cs.maxBitsPerChar()
+ def maxBytesPerChar(): Float = cs.maxBytesPerChar()
+
+ /**
+ * Hyjack a JavaCharset to drive our code.
+ *
+ * We want to avoid re-implementing all the error handling and flush/end logic.
+ */
+ protected object ProxyJavaCharset
+ extends JavaCharset("proxyCharset", Array()) {
+ override def newDecoder(): JavaCharsetDecoder =
+ new ProxyJavaCharsetDecoder(this, new NBitsWidth_BitsCharsetDecoder(cs))
+ override def newEncoder(): JavaCharsetEncoder =
+ new ProxyJavaCharsetEncoder(this, new NBitsWidth_BitsCharsetEncoder(cs, cs.replacementCharCode))
+ override def contains(jcs: JavaCharset): Boolean =
+ Assert.usageError("contains method not to be called on " + this)
+ }
- def cs: NBitsWidthCharset
- final def bitWidthOfACodeUnit = cs.bitWidthOfACodeUnit
- final def requiredBitOrder = cs.requiredBitOrder
}
/**
+ * Hyjack a JavaCharsetDecoder to drive the decodeLoop.
+ *
+ * This avoids us reimplementing all the error handling and flush/end logic.
+ */
+protected final class ProxyJavaCharsetDecoder(cs: JavaCharset, real: NBitsWidth_BitsCharsetDecoder)
+ extends JavaCharsetDecoder(cs, 1.0F, 1.0F) {
+ override def decodeLoop(in: ByteBuffer, out: CharBuffer): CoderResult =
+ real.decodeLoop(in, out)
+}
+
+/**
+ * Hyjack a JavaCharsetEncoder to drive the encodeLoop.
+ *
+ * This avoids us reimplementing all the error handling and flush/end logic.
+ */
+protected final class ProxyJavaCharsetEncoder(cs: JavaCharset, real: NBitsWidth_BitsCharsetEncoder)
+ extends JavaCharsetEncoder(cs, 1.0F, 1.0F) {
+ override def encodeLoop(in: CharBuffer, out: ByteBuffer): CoderResult =
+ real.encodeLoop(in, out)
+}
+/**
* You have to initialize one of these for a specific ByteBuffer because
* the encoding is N-bits wide, so we need additional state beyond just
* the byte position and limit that a ByteBuffer provides in order to
* properly sequence through the data.
*/
-class NBitsWidthCharsetDecoder(override val cs: NBitsWidthCharset)
- extends java.nio.charset.CharsetDecoder(cs,
- 8.0F / cs.bitWidthOfACodeUnit.toFloat, // average
- 8.0F / cs.bitWidthOfACodeUnit.toFloat) // maximum.
- with NonByteSizeCharsetDecoder
- with NBitsWidthEncoderDecoderMixin {
-
- def output(charCode: Int, out: CharBuffer) {
+final class NBitsWidth_BitsCharsetDecoder(override protected val cs: NBitsWidth_BitsCharset)
+ extends BitsCharsetDecoder
+ with NBitsWidth_BitsEncoderDecoderMixin {
+ /*
+ * We delegate all the normal charset behavior used to customize and control
+ * errors of a decoder to the proxy decoder.
+ *
+ * The proxy decoder calls back to this object to do the decodeLoop.
+ */
+ def malformedInputAction(): CodingErrorAction = proxy.malformedInputAction()
+ def unmappableCharacterAction(): CodingErrorAction = proxy.unmappableCharacterAction()
+ def onMalformedInput(action: CodingErrorAction) = { proxy.onMalformedInput(action); this }
+ def onUnmappableCharacter(action: CodingErrorAction) = { proxy.onUnmappableCharacter(action); this }
+ def replacement() = proxy.replacement()
+ def replaceWith(newReplacement: String) = { proxy.replaceWith(newReplacement); this }
+
+ def decode(in: ByteBuffer, out: CharBuffer, endOfInput: Boolean) = proxy.decode(in, out, endOfInput)
+ def flush(out: CharBuffer) = proxy.flush(out)
+
+ private val proxy = new ProxyJavaCharsetDecoder(ProxyJavaCharset, this)
+
+ private def output(charCode: Int, out: CharBuffer) {
out.put(cs.codeToChar(charCode))
}
+
+ private var startBitOffset0b = 0
+ private var startBitOffsetHasBeenSet = false
+ private var startBitOffsetHasBeenUsed = false
+
+ /**
+ * Stores number of bits in the fragment byte at the end of the data.
+ *
+ * The frag byte value is an additional byte 1 past the end of the
+ * byte buffer's last active position. That is, it is at the limit location.
+ *
+ * This limit location must exist. I.e., the limit must be less than the capacity.
+ */
+ private var maybeBitLimitOffset0b: MaybeULong = MaybeULong.Nope
+ private var priorByte: MaybeInt = MaybeInt.Nope
+ private var fragByte_ : MaybeInt = MaybeInt.Nope // populated from bytebuffer's final fragment byte if there is one.
+
+ def reset() = {
+ resetStartBit()
+ maybeBitLimitOffset0b = MaybeULong.Nope
+ priorByte = MaybeInt.Nope
+ priorByteBitCount = 0
+ fragByte_ = MaybeInt.Nope
+ proxy.reset()
+ isReset = true
+ this
+ }
+
+ private def getFragByte(in: ByteBuffer): Int = {
+ Assert.usage(this.maybeBitLimitOffset0b.isDefined &&
+ this.maybeBitLimitOffset0b.get > 0)
+ Assert.usage(in.limit() < in.capacity()) // room for 1 more byte.
+ Assert.usage(in.remaining() == 0) // only call this after consuming all bytes.
+ if (fragByte_.isEmpty) {
+ val savedLimit = in.limit()
+ in.limit(savedLimit + 1)
+ val finalByte = in.get(savedLimit)
+ in.limit(savedLimit)
+ fragByte_ = MaybeInt(finalByte)
+ }
+ fragByte_.get
+ }
+
+ /**
+ * Bit-oriented decoders have to be passed a standard byteBuffer and CharBuffer
+ * via the normal Java decodeLoop API. However, as we are bit oriented, there's the
+ * possibility that there is a fragment of a byte at the end, and there
+ * can be an offset within the first byte for the start.
+ *
+ * So to get that information over to the decoder, given that the decodeLoop
+ * API doesn't pass it, we have setters.
+ *
+ * So a decoder is stateful. They're always stateful, so this isn't changing
+ * that. We're just adding yet more state.
+ */
+
+ final def setInitialBitOffset(bitOffset0to7: Int) {
+ Assert.usage(!startBitOffsetHasBeenSet, "Already set. Cannot set again until decoder is reset().")
+ Assert.usage(bitOffset0to7 <= 7 && bitOffset0to7 >= 0)
+ startBitOffset0b = bitOffset0to7
+ startBitOffsetHasBeenSet = true
+ }
+
+ /**
+ * If set, then it means there IS one more byte (at least) at the end of
+ * the ByteBuffer which holds the fragment byte. This final byte is past the
+ * limit of the ByteBuffer.
+ */
+ final def setFinalByteBitLimitOffset0b(bitLimitOffset0b: MaybeULong) {
+ maybeBitLimitOffset0b = bitLimitOffset0b
+ }
+
+ final protected def getFinalByteBitLimitOffset0b() = maybeBitLimitOffset0b
+
+ /**
+ * Calling this returns the startBitOffset0b, but also sets flags so
+ * that it won't be processed again. It's a once-only thing until a reset.
+ */
+ final protected def takeStartBitOffset() = {
+ if (startBitOffsetHasBeenUsed) 0 // one time we return the value. After that 0 until a reset.
+ else {
+ startBitOffsetHasBeenUsed = true
+ startBitOffset0b
+ }
+ }
+
+ final protected def resetStartBit() {
+ startBitOffsetHasBeenUsed = false
+ startBitOffset0b = 0
+ startBitOffsetHasBeenSet = false
+ }
+
+ /**
+ * When there is a prior byte, this gives the number of bits from it
+ * that have not yet been consumed. Zero when there is no prior byte.
+ *
+ * Value ranges from 1 to 7.
+ *
+ * There is a case where it is set to 8, which is when we put
+ * a new whole byte from the input into it, and then immediately
+ * remove a charcode from it.
+ */
+ private var priorByteBitCount = 0
+
+ /*
+ * These predicates make reasoning about the various combinations
+ * easier.
+ *
+ * For the input, we can have data (whole bytes), a fragment of a byte,
+ * or nothing.
+ *
+ * For the output, we either have room for a character, nor not.
+ *
+ * For the decoder itself, we have whether there is a prior byte
+ * of data (generally part of a byte) or not.
+ */
+
+ /**
+ * No whole bytes are available
+ */
+ @inline private final def NoData(in: ByteBuffer) = !YesData(in) && NoFrag
+
+ /**
+ * At least 1 whole byte is available
+ */
+ @inline private final def YesData(in: ByteBuffer) = in.hasRemaining()
+
+ /**
+ * Space for at least 1 character of output
+ */
+ @inline private final def YesSpace(out: CharBuffer) = out.hasRemaining()
+
+ /**
+ * No partially consumed prior byte
+ */
+ private final def NoPrior = {
+ val res = !YesPrior
+ if (res) Assert.invariant(priorByteBitCount == 0)
+ res
+ }
+
+ /**
+ * There is a partially consumed prior byte. Contains at least 1 bit.
+ *
+ * When we first create a priorByte it may contain 8 bits, but that
+ * is a transient condition. When this is called it should be 1 to 7
+ */
+ private final def YesPrior = {
+ val res = priorByte.isDefined
+ if (res) Assert.invariant(priorByteBitCount > 0 && priorByteBitCount <= 7)
+ res
+ }
+
+ @inline private def takeCharCodeFromPriorByte() =
+ takeBitsFromPriorByte(bitWidthOfACodeUnit)
+
+ private def takeBitsFromPriorByte(nBits: Int) = {
+ Assert.usage(nBits > 0)
+ Assert.invariant(priorByte.isDefined)
+ val cnt = priorByteBitCount
+ Assert.invariant(cnt >= nBits)
+ val pb = priorByte.get
+ val mask = 0xFF >> (8 - nBits)
+ val isolatingShift =
+ if (this.requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
+ cnt - nBits
+ } else {
+ // LSBF
+ 8 - cnt
+ }
+ val charCode =
+ (pb >>> isolatingShift) & mask
+ priorByteBitCount = priorByteBitCount - nBits
+ if (priorByteBitCount == 0)
+ priorByte = MaybeInt.Nope
+ charCode
+ }
+
+ @inline private def takeCharCodeFromFragByte(in: ByteBuffer): Int =
+ takeBitsFromFragByte(in, bitWidthOfACodeUnit)
+
+ private def takeBitsFromFragByte(in: ByteBuffer, nBits: Int): Int = {
+ val fb = getFragByte(in)
+ val nBitsInFrag = this.maybeBitLimitOffset0b.get
+ Assert.invariant(nBitsInFrag >= nBits)
+ var bits = 0
+ var newFB = 0
+ val mask = 0xFF >> (8 - nBits)
+ if (this.requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
+ // MSBF
+ bits = (fb >>> (8 - nBits)) & mask
+ newFB = (fb << nBits) & 0xFF
+ } else {
+ // LSBF
+ bits = fb & mask
+ newFB = (fb >>> nBits) & 0xFF
+ }
+ val newLimit = nBitsInFrag - nBits
+ if (newLimit > 0)
+ maybeBitLimitOffset0b = MaybeULong(newLimit)
+ else
+ maybeBitLimitOffset0b = MaybeULong.Nope
+ fragByte_ =
+ if (maybeBitLimitOffset0b.isEmpty)
+ MaybeInt.Nope
+ else
+ MaybeInt(newFB)
+ bits
+ }
+
+ /**
+ * No partial fragment byte at the end of the data (after all whole bytes)
+ */
+ private def NoFrag = {
+ val res = maybeBitLimitOffset0b.isEmpty
+ if (res)
+ Assert.invariant(fragByte_.isEmpty)
+ res
+ }
+
+ /**
+ * There is a fragment byte after all whole bytes. Contains at least 1 bit.
+ */
+ private def YesFrag(in: ByteBuffer) = {
+ val res = maybeBitLimitOffset0b.isDefined && !in.hasRemaining()
+ if (res) {
+ val lim = maybeBitLimitOffset0b.get
+ if (lim == 0) Assert.invariant(fragByte_.isEmpty)
+ Assert.invariant(lim >= 0 && lim <= 7)
+ }
+ res
+ }
+
+ /**
+ * Returns false if not enough bits are in the data to handle the start offset.
+ */
+ private def handleStartOffset(in: ByteBuffer, out: CharBuffer): Boolean = {
+ val bitOffset = takeStartBitOffset()
+ if (bitOffset != 0) {
+ Assert.invariant(NoPrior)
+ // if there's a start offset, then we're just starting.
+ // we have to offset into the first byte if there is one.
+ if (YesData(in)) {
+ priorByte = MaybeInt(Bits.asUnsignedByte(in.get()))
+ priorByteBitCount = 8
+ this.takeBitsFromPriorByte(bitOffset)
+ // fall through to main loop below, but now we have a prior byte, and no start offset.
+ true
+ } else if (YesFrag(in)) {
+ // no whole bytes of data. But there could still be a frag byte
+ if (this.maybeBitLimitOffset0b.get >= bitOffset) {
+ // we have to have enough bits in the frag.
+ // TODO: TBD should the above be a return UNDERFLOW instead of abort?
+ takeBitsFromFragByte(in, bitOffset)
+ true
+ } else false
+ } else {
+ false
+ }
+ } else
+ true
+ }
+
+ final def decodeLoop(in: ByteBuffer, out: CharBuffer): CoderResult = {
+ isReset = false
+
+ //
+ // Now we have to adjust for the starting bit offset
+ // Grab first byte so we can start at an offset into it.
+ //
+ if (!handleStartOffset(in, out)) {
+ CoderResult.UNDERFLOW // not enough data to handle the starting offset
+ } else {
+ //
+ // Once around this loop per character decoded.
+ //
+ // Note that we can enter this loop at any state. E.g., we could have pulled
+ // a byte from the input, decoded some bits from it (to go with earlier bits)
+ // but still have more to go in that character, but have to return from this
+ // loop due to charBuf overflow. When we re-enter this loop we
+ // presumably will have more charBuf space, and we have to pick up where
+ // we left off.
+ //
+ // States:
+ // YesPrior or NoPrior - byte currently being consumed
+ // Output: YesSpace or NoSpace
+ // Input: YesData (more whole bytes), YesFrag (partial final byte), NoData (no more data of any kind)
+ //
+ var rc: Maybe[CoderResult] = Maybe.Nope
+ while (rc.isEmpty) {
+ rc = attemptToDecodeOne(in, out)
+ }
+ rc.get
+ }
+ }
+
+ /**
+ * Attempt to decode a single character.
+ *
+ * Returns Maybe.Nope to indicate successful decode.
+ * Returns One(CoderResult.UNDERFLOW) or One(CoderResult.OVERFLOW) to
+ * indicate it was unable to decode.
+ */
+ private val underflow = Maybe(CoderResult.UNDERFLOW)
+ private val overflow = Maybe(CoderResult.OVERFLOW)
+
+ // package private - so we can call for unit testing. But this is not
+ // part of the public interface of these decoders.
+ //
+ private[charset] def attemptToDecodeOne(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
+ if (YesPrior)
+ attemptToDecodeOneYesPrior(in, out)
+ else
+ attemptToDecodeOneNoPrior(in, out)
+ }
+
+ private def attemptToDecodeOneYesPrior(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
+ //
+ // YesPrior Cases - where there is a prior byte, and it has enough
+ // bits to produce a character just from there.
+ //
+ if (priorByteBitCount >= bitWidthOfACodeUnit) {
+ if (YesSpace(out)) {
+ // there is space in the charbuffer, produce a character
+ val charCode = this.takeCharCodeFromPriorByte() // priorByteBitCount gets smaller.
+ output(charCode, out)
+ Maybe.Nope
+ } else {
+ overflow
+ }
+ } else //
+ //
+ // YesPrior Cases - but prior byte alone is not enough, but YesSpace
+ //
+ if (YesSpace(out)) {
+ attemptToDecodeOneYesPriorYesSpace(in, out)
+ } else {
+ attemptToDecodeOneYesPriorNoSpace(in, out)
+ }
+ }
+
+ private def attemptToDecodeOneYesPriorYesSpace(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
+ if (YesData(in))
+ attemptToDecodeOneYesPriorYesSpaceYesData(in, out)
+ else if (YesFrag(in))
+ attemptToDecodeOneYesPriorYesSpaceYesFrag(in, out)
+ else if (NoData(in))
+ attemptToDecodeOneYesPriorYesSpaceNoData(in, out)
+ else
+ Assert.impossible("All cases should have been exhausted")
+ }
+
+ private lazy val mask = 0xFF >>> (8 - bitWidthOfACodeUnit)
+
+ def attemptToDecodeOneYesPriorYesSpaceYesData(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
+ // we need more bits from the next byte. At least 1 bit comes from
+ // priorByte. We will produce a character and go around again, because
+ // a whole byte from input always provides enough bits.
+ val nNeededBits = bitWidthOfACodeUnit - priorByteBitCount
+ Assert.invariant(nNeededBits >= 1)
+ val currentByte = Bits.asUnsignedByte(in.get())
+ val nPriorBits = priorByteBitCount
+ val priorBits = takeBitsFromPriorByte(priorByteBitCount)
+ val nCurrentBits = (bitWidthOfACodeUnit - nPriorBits)
+
+ val charCode =
+ if (requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
+ val priorBitsInPosition = priorBits << nCurrentBits
+ val currentBits = currentByte >>> (8 - nCurrentBits)
+ val cc = (priorBitsInPosition | currentBits) & mask
+ cc
+ } else {
+ // LSBF
+ val priorBitsInPosition = priorBits
+ val currentBits = (currentByte << nPriorBits) & mask
+ val cc = (priorBitsInPosition | currentBits) & mask
+ cc
+ }
+ priorByte = MaybeInt(currentByte)
+ priorByteBitCount = 8 - nCurrentBits // priorByte will become the unconsumed bits from data byte.
+ output(charCode, out)
+ Maybe.Nope
+ }
+
+ def attemptToDecodeOneYesPriorYesSpaceYesFrag(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
+ if (priorByteBitCount + this.maybeBitLimitOffset0b.get >= bitWidthOfACodeUnit) {
+ // Case where combining prior with frag gets us enough bits for a character
+ // We will produce a character and go around again.
+ val nPriorBits = priorByteBitCount
+ val priorBits = takeBitsFromPriorByte(priorByteBitCount)
+ val nCurrentBits = (bitWidthOfACodeUnit - nPriorBits)
+ val currentBits = takeBitsFromFragByte(in, nCurrentBits)
+ val charCode =
+ if (requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
+ val priorBitsInPosition = priorBits << nCurrentBits
+ val cc = (priorBitsInPosition | currentBits) & mask
+ cc
+ } else {
+ // LSBF
+ val currentBitsInPosition = currentBits << nPriorBits
+ val cc = (priorBits | currentBitsInPosition) & mask
+ cc
+ }
+ priorByte = MaybeInt.Nope
+ priorByteBitCount = 0
+ output(charCode, out)
+ // No more prior byte, and frag byte gets smaller.
+ Maybe.Nope
+ } else {
+ // Not enough bits left period combining prior and frag.
+ underflow
+ }
+ }
+
+ def attemptToDecodeOneYesPriorYesSpaceNoData(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
+ //
+ // Keep in mind the invariant. There is NOT enoug bits in the prior for a character
+ //
+ Assert.invariant(priorByteBitCount < bitWidthOfACodeUnit)
+ underflow
+ }
+
+ private def attemptToDecodeOneNoPrior(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
+ if (YesSpace(out))
+ attemptToDecodeOneNoPriorYesSpace(in, out)
+ else
+ attemptToDecodeOneNoPriorNoSpace(in, out)
+ }
+
+ private def attemptToDecodeOneNoPriorYesSpace(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
+ //
+ //////////////////////////////////
+ // NoPrior Cases - but YesSpace
+ //////////////////////////////////
+ //
+ if (YesData(in)) {
+ //
+ // Fresh start or we hit a clean byte boundary
+ //
+ priorByte = MaybeInt(Bits.asUnsignedByte(in.get()))
+ priorByteBitCount = 8
+ val currentCharCode = takeCharCodeFromPriorByte()
+ output(currentCharCode, out)
+ Maybe.Nope
+ } else if (YesFrag(in)) {
+ // There is a partial frag byte available in the data, and that may provide
+ // sufficient bits to decode a character. If enough, decode one & loop
+ // reducing the size of the frag.
+ //
+ val fragLen = maybeBitLimitOffset0b.get
+ if (fragLen >= bitWidthOfACodeUnit) {
+ val charCode = this.takeCharCodeFromFragByte(in)
+ output(charCode, out)
+ // frag gets smaller, or disappears.
+ Maybe.Nope
+ } else {
+ // frag is insufficient to create a character
+ underflow
+ }
+ } else if (NoData(in)) {
+ underflow
+ } else {
+ Assert.impossible("All cases are supposed to have been exhausted.")
+ }
+ }
+
+ private def attemptToDecodeOneNoPriorNoSpace(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
+ //
+ //////////////////////////////////
+ // NoPrior, but NoSpace
+ //
+ // We'll never produce a character here
+ // We always return with some CoderResult status.
+ //////////////////////////////////
+ if (YesData(in)) {
+ overflow
+ } else if (YesFrag(in)) {
+ val fragLen = maybeBitLimitOffset0b.get
+ if (fragLen >= bitWidthOfACodeUnit) {
+ // if frag has enough for a character, then overflow.
+ overflow
+ } else {
+ underflow
+ }
+ } else if (NoData(in)) {
+ underflow
+ } else {
+ Assert.impossible("All cases are supposed to have been exhausted.")
+ }
+ }
+
+ private def attemptToDecodeOneYesPriorNoSpace(in: ByteBuffer, out: CharBuffer): Maybe[CoderResult] = {
+ ///////////////////////////////
+ // YesPrior but NoSpace Cases
+ ///////////////////////////////
+ if (YesData(in)) {
+ overflow
+ } else if (YesFrag(in)) {
+ if (priorByteBitCount + this.maybeBitLimitOffset0b.get >= bitWidthOfACodeUnit) {
+ overflow
+ } else {
+ // Not enough bits left combining prior and frag.
+ // Note that we prefer underflow to overflow. There's no
+ // character to produce here, so we haven't run out of space in the
+ // output, we're unable to produce one.
+ underflow
+ }
+ } else if (NoData(in)) {
+ underflow
+ } else {
+ Assert.impossible("All cases are supposed to have been exhausted.")
+ }
+ }
}
-class NBitsWidthCharsetEncoder(override val cs: NBitsWidthCharset,
- override val replacementChar: Int)
- extends java.nio.charset.CharsetEncoder(cs,
- cs.bitWidthOfACodeUnit.toFloat / 8.0F, // average
- 1.0F) // maximum. Note can't use bitWidth/8.0 here because CharsetEncoder base class requires it to be 1 or greater.) // maximum
- with NonByteSizeCharsetEncoder
- with NBitsWidthEncoderDecoderMixin {
+class NBitsWidth_BitsCharsetEncoder(override protected val cs: NBitsWidth_BitsCharset,
+ replacementChar: Int)
+ extends BitsCharsetEncoder
+ with NBitsWidth_BitsEncoderDecoderMixin {
+
+ /*
+ * We delegate all the normal charset behavior used to customize and control
+ * errors of an encoder to the proxy encoder.
+ *
+ * The proxy encoder calls back to this object to do the encodeLoop.
+ */
+ def malformedInputAction(): CodingErrorAction = proxy.malformedInputAction()
+ def unmappableCharacterAction(): CodingErrorAction = proxy.unmappableCharacterAction()
+ def onMalformedInput(action: CodingErrorAction) = { proxy.onMalformedInput(action); this }
+ def onUnmappableCharacter(action: CodingErrorAction) = { proxy.onUnmappableCharacter(action); this }
+ def replacement() = proxy.replacement()
+ def replaceWith(newReplacement: Array[Byte]) = { proxy.replaceWith(newReplacement); this }
+
+ def encode(in: CharBuffer, out: ByteBuffer, endOfInput: Boolean) = proxy.encode(in, out, endOfInput)
+ def flush(out: ByteBuffer) = proxy.flush(out)
+
+ private lazy val proxy = new ProxyJavaCharsetEncoder(ProxyJavaCharset, this)
+
+ private var partialByte: Int = 0
+ private var partialByteLenInBits: Int = 0
+
+ def reset() = {
+ partialByte = 0
+ partialByteLenInBits = 0
+ proxy.reset()
+ isReset = true
+ this
+ }
+
+ def encodeLoop(in: CharBuffer, out: ByteBuffer): CoderResult = {
+ isReset = false
+ while (true) {
+ val inHasRemainingData = in.hasRemaining()
+ val outHasRemainingSpace = out.hasRemaining()
+
+ if (!inHasRemainingData) {
+ if (partialByteLenInBits > 0) {
+ if (!outHasRemainingSpace) {
+ // no remaining input, but there's a partial byte we need to write,
+ // and nowhere to write it to
+ return CoderResult.OVERFLOW
+ } else {
+ // no remaining input, but there's a partial byte, write the partial
+ // byte (includes padding) and finish
+ out.put(partialByte.toByte)
+ partialByte = 0
+ partialByteLenInBits = 0
+ return CoderResult.UNDERFLOW
+ }
+ } else {
+ // no remaining input, and no partial byte, nothing left to
+ // encode/write, finish
+ return CoderResult.UNDERFLOW
+ }
+ } else {
+ if (!outHasRemainingSpace) {
+ // there's input to encode, but nowhere to write it to
+ return CoderResult.OVERFLOW
+ }
+
+ // there's data to encode and enough space to write a byte, encode the
+ // character
+
+ val charCode = charToCharCode(in.get())
+
+ val charCodeToWrite =
+ if (charCode.isDefined) {
+ charCode.get
+ } else {
+ // character must fit in the bit width of a code unit, unmappable error
+ val unmappableAction = unmappableCharacterAction
+ if (unmappableAction == CodingErrorAction.REPLACE) {
+ // CharsetEncoder, which handles character replacement, assumes
+ // that the replacement character is made up of full bytes. That
+ // isn't the case for this charset, so we will just manually
+ // replace the character ourselves and not let CharsetEncoder
+ // know there was ever a problem.
+ replacementChar
+ } else {
+ Assert.invariant(unmappableAction == CodingErrorAction.REPORT)
+ // Just report the unmappable character. Note that we should back
+ // up the position of the CharBuffer since we read a character
+ // but it was invalid. The charset encoder that called encodeLoop
+ // will handle skipping the invalid character. Note that the
+ // character is only skipped in CodingErrorAction.IGNORE and
+ // REPLACE. We handle REPLACE ourselves, and IGNORE isn't
+ // supported by DFDL, so this *shouldn't* matter, but it might
+ // someday if IGNORE is supported.
+ in.position(in.position() - 1)
+ return CoderResult.unmappableForLength(1)
+ }
+ }
+
+ if (partialByteLenInBits == 0) {
+ // no partial byte exists, make this the partial byte
+ if (requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
+ partialByte = charCodeToWrite << (8 - bitWidthOfACodeUnit)
+ } else {
+ partialByte = charCodeToWrite
+ }
+ partialByteLenInBits = bitWidthOfACodeUnit
+ } else if ((partialByteLenInBits + bitWidthOfACodeUnit) >= 8) {
+ // This character's bits will fill up the byte. Some bits might be left over.
+ // There's a partial byte, add enough bits to make it a full byte and
+ // write it, then save whatever is remaining (could be 0 bits
+ // remaining) as a partial byte
+ val nUsedBits = 8 - partialByteLenInBits
+ if (requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
+ val nLeftOverBits = bitWidthOfACodeUnit - nUsedBits
+ Assert.invariant(nLeftOverBits >= 0)
+ partialByte |= (charCodeToWrite >> nLeftOverBits) & 0xFF
+ out.put(partialByte.toByte)
+ val leftOverMask = (1 << nLeftOverBits) - 1
+ val newPartialByte = (charCodeToWrite & leftOverMask) << (8 - nLeftOverBits)
+ partialByte = newPartialByte & 0xFF
+ partialByteLenInBits = nLeftOverBits
+ } else {
+ // LSBF
+ partialByte |= (charCodeToWrite << partialByteLenInBits) & 0xFF
+ out.put(partialByte.toByte)
+ partialByte = charCodeToWrite >> nUsedBits
+ partialByteLenInBits = bitWidthOfACodeUnit - nUsedBits
+ }
+ } else {
+ // there's a partial byte but there won't be enough bits to make it a full byte
+ if (requiredBitOrder eq BitOrder.MostSignificantBitFirst) {
+ partialByte |= (charCodeToWrite << (8 - partialByteLenInBits - bitWidthOfACodeUnit)) & 0xFF
+ } else {
+ // LSBF
+ partialByte |= (charCodeToWrite << partialByteLenInBits) & 0xFF
+ }
+ partialByteLenInBits = bitWidthOfACodeUnit + partialByteLenInBits
+ }
+
+ //
+ // Verify invariant that the unused bits of the partialByte are always 0
+ //
+ Assert.invariant({
+ val unusedPartialByteMask: Int =
+ if (requiredBitOrder eq BitOrder.MostSignificantBitFirst)
+ (1 << (8 - partialByteLenInBits)) - 1
+ else
+ (-1 << partialByteLenInBits) & 0xFF
+ val unusedPartialByteBits = partialByte & unusedPartialByteMask
+ unusedPartialByteBits == 0
+ })
+ }
+ }
+
+ Assert.impossible("Incorrect return from encodeLoop")
+ }
def charToCharCode(char: Char): MaybeInt = {
cs.charToCode(char)
}
-
}
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/OctalLSBF3BitDecoder.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/OctalLSBF3BitDecoder.scala
index aeff493..d85f9c4 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/OctalLSBF3BitDecoder.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/OctalLSBF3BitDecoder.scala
@@ -43,14 +43,14 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
*/
object OctalLSBF3BitCharset
- extends NBitsWidthCharset("X-DFDL-OCTAL-LSBF",
+ extends NBitsWidth_BitsCharset("X-DFDL-OCTAL-LSBF",
"01234567",
3, // width
BitOrder.LeastSignificantBitFirst,
0) // replacement charCode for encoding of unmapped chars.
object OctalMSBF3BitCharset
- extends NBitsWidthCharset("X-DFDL-OCTAL-MSBF",
+ extends NBitsWidth_BitsCharset("X-DFDL-OCTAL-MSBF",
"01234567",
3, // width
BitOrder.MostSignificantBitFirst,
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/USASCII6BitPackedDecoder.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/USASCII6BitPackedDecoder.scala
index bafdcf6..d35687a 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/USASCII6BitPackedDecoder.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/USASCII6BitPackedDecoder.scala
@@ -43,7 +43,7 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
*/
object USASCII6BitPackedCharset
- extends NBitsWidthCharset("X-DFDL-US-ASCII-6-BIT-PACKED",
+ extends NBitsWidth_BitsCharset("X-DFDL-US-ASCII-6-BIT-PACKED",
"""@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_ !"#$%&'()*+,-./0123456789:;<=>?""",
6, // width
BitOrder.LeastSignificantBitFirst,
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/USASCII7BitPackedDecoder.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/USASCII7BitPackedDecoder.scala
index 73ddb8f..d8e3c72 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/USASCII7BitPackedDecoder.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/USASCII7BitPackedDecoder.scala
@@ -46,7 +46,7 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
*/
object USASCII7BitPackedCharset
- extends NBitsWidthCharset("X-DFDL-US-ASCII-7-BIT-PACKED",
+ extends NBitsWidth_BitsCharset("X-DFDL-US-ASCII-7-BIT-PACKED",
(0 to 127).map { _.toChar }.mkString,
7,
BitOrder.LeastSignificantBitFirst,
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/X_DFDL_6_BIT_DFI_264_DUI_001.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/X_DFDL_6_BIT_DFI_264_DUI_001.scala
index 74b988c..6644bc8 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/X_DFDL_6_BIT_DFI_264_DUI_001.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/X_DFDL_6_BIT_DFI_264_DUI_001.scala
@@ -43,7 +43,7 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
*/
object X_DFDL_6_BIT_DFI_264_DUI_001
- extends NBitsWidthCharset("X-DFDL-6-BIT-DFI-264-DUI-001",
+ extends NBitsWidth_BitsCharset("X-DFDL-6-BIT-DFI-264-DUI-001",
" 123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD0",
6, // width
BitOrder.LeastSignificantBitFirst,
diff --git a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/X_DFDL_BITS.scala b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/X_DFDL_BITS.scala
index 44cb265..8ff4806 100644
--- a/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/X_DFDL_BITS.scala
+++ b/daffodil-io/src/main/scala/edu/illinois/ncsa/daffodil/processors/charset/X_DFDL_BITS.scala
@@ -41,14 +41,14 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
*/
object X_DFDL_BITS_LSBF
- extends NBitsWidthCharset("X-DFDL-BITS-LSBF",
+ extends NBitsWidth_BitsCharset("X-DFDL-BITS-LSBF",
"01",
1,
BitOrder.LeastSignificantBitFirst,
0)
object X_DFDL_BITS_MSBF
- extends NBitsWidthCharset("X-DFDL-BITS-MSBF",
+ extends NBitsWidth_BitsCharset("X-DFDL-BITS-MSBF",
"01",
1,
BitOrder.MostSignificantBitFirst,
diff --git a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/FormatInfoForUnitTest.scala b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/FormatInfoForUnitTest.scala
index fee4a94..e3742e1 100644
--- a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/FormatInfoForUnitTest.scala
+++ b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/FormatInfoForUnitTest.scala
@@ -1,9 +1,6 @@
package edu.illinois.ncsa.daffodil.io
-import java.nio.charset.Charset
import java.nio.charset.CodingErrorAction
-import java.nio.charset.CharsetEncoder
-import java.nio.charset.CharsetDecoder
import edu.illinois.ncsa.daffodil.util.Maybe
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BinaryFloatRep
@@ -11,6 +8,11 @@ import edu.illinois.ncsa.daffodil.util.MaybeInt
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.ByteOrder
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.UTF16Width
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.EncodingErrorPolicy
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharset
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharsetDecoder
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharsetEncoder
+import edu.illinois.ncsa.daffodil.processors.charset.StandardBitsCharsets
+import edu.illinois.ncsa.daffodil.processors.charset.NBitsWidth_BitsCharset
object FormatInfoForUnitTest {
def apply() = {
@@ -22,13 +24,13 @@ object FormatInfoForUnitTest {
class FormatInfoForUnitTest private ()
extends FormatInfo {
- var priorEncoding: Charset = java.nio.charset.StandardCharsets.UTF_8
+ var priorEncoding: BitsCharset = StandardBitsCharsets.UTF_8
- var encoder: CharsetEncoder = priorEncoding.newEncoder()
- var decoder: CharsetDecoder = priorEncoding.newDecoder()
- var reportingDecoder: CharsetDecoder = _
+ var encoder: BitsCharsetEncoder = priorEncoding.newEncoder()
+ var decoder: BitsCharsetDecoder = priorEncoding.newDecoder()
+ var reportingDecoder: BitsCharsetDecoder = _
- var replacingDecoder: CharsetDecoder = _
+ var replacingDecoder: BitsCharsetDecoder = _
var byteOrder: ByteOrder = ByteOrder.BigEndian
var bitOrder: BitOrder = BitOrder.MostSignificantBitFirst
@@ -39,7 +41,7 @@ class FormatInfoForUnitTest private ()
var encodingMandatoryAlignmentInBits: Int = 8
var encodingErrorPolicy: EncodingErrorPolicy = EncodingErrorPolicy.Replace
- def reset(cs: Charset): Unit = {
+ def reset(cs: BitsCharset): Unit = {
priorEncoding = cs
init()
}
@@ -64,7 +66,7 @@ class FormatInfoForUnitTest private ()
d
}
priorEncoding match {
- case decoderWithBits: NonByteSizeCharset => {
+ case decoderWithBits: NBitsWidth_BitsCharset => {
encodingMandatoryAlignmentInBits = 1
maybeCharWidthInBits = MaybeInt(decoderWithBits.bitWidthOfACodeUnit)
}
diff --git a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestByteBufferDataInputStream.scala b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestByteBufferDataInputStream.scala
index aec214a..9961d57 100644
--- a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestByteBufferDataInputStream.scala
+++ b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestByteBufferDataInputStream.scala
@@ -35,7 +35,6 @@ package edu.illinois.ncsa.daffodil.io
import java.nio.ByteBuffer
import java.nio.CharBuffer
import java.nio.charset.MalformedInputException
-import java.nio.charset.StandardCharsets
import java.util.regex.Pattern
import scala.BigInt
@@ -55,6 +54,7 @@ import edu.illinois.ncsa.daffodil.util.MaybeULong
import edu.illinois.ncsa.daffodil.util.Misc
import passera.unsigned.ULong
import java.nio.charset.CodingErrorAction
+import edu.illinois.ncsa.daffodil.processors.charset.StandardBitsCharsets
class TestByteBufferDataInputStream {
val tenDigits = "1234567890"
@@ -704,7 +704,7 @@ class TestByteBufferDataInputStream {
val data = data1 ++ badByte ++ data3
val dis = ByteBufferDataInputStream(data)
val finfo = FormatInfoForUnitTest()
- finfo.reset(StandardCharsets.UTF_16BE)
+ finfo.reset(StandardBitsCharsets.UTF_16BE)
val pattern = Pattern.compile(".{1,13}")
val matcher = pattern.matcher("")
val isMatch = dis.lookingAt(matcher, finfo)
diff --git a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestByteBufferDataInputStream3Bit.scala b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestByteBufferDataInputStream3Bit.scala
index e2a58d5..28c817d 100644
--- a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestByteBufferDataInputStream3Bit.scala
+++ b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestByteBufferDataInputStream3Bit.scala
@@ -43,6 +43,7 @@ import edu.illinois.ncsa.daffodil.exceptions.Assert
import edu.illinois.ncsa.daffodil.util.MaybeULong
import edu.illinois.ncsa.daffodil.util.Misc
import edu.illinois.ncsa.daffodil.processors.charset.OctalLSBF3BitCharset
+import edu.illinois.ncsa.daffodil.processors.charset.NBitsWidth_BitsCharsetEncoder
/**
* tests of 7-bit characters
@@ -82,8 +83,8 @@ class TestByteBufferDataInputStream3Bit {
bb.flip()
val res = (0 to bb.limit() - 1).map { bb.get(_) }
// val bitsAsString = Misc.bytes2Bits(res.toArray)
- val enc = encoder.asInstanceOf[NonByteSizeCharsetEncoder]
- val nBits = s.length * enc.bitWidthOfACodeUnit
+ val enc = encoder.asInstanceOf[NBitsWidth_BitsCharsetEncoder]
+ val nBits = s.length * enc.bitsCharset.bitWidthOfACodeUnit
val bitStrings = res.map { b => (b & 0xFF).toBinaryString.reverse.padTo(8, '0').reverse }.toList
val allBits = bitStrings.reverse.mkString.takeRight(nBits)
val threeBitChunks = allBits.reverse.sliding(3, 3).map { _.reverse }.toList.reverse.mkString
diff --git a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestByteBufferDataInputStream7.scala b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestByteBufferDataInputStream7.scala
index 810ec1e..18e1b62 100644
--- a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestByteBufferDataInputStream7.scala
+++ b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestByteBufferDataInputStream7.scala
@@ -43,6 +43,7 @@ import edu.illinois.ncsa.daffodil.exceptions.Assert
import edu.illinois.ncsa.daffodil.processors.charset.USASCII7BitPackedCharset
import edu.illinois.ncsa.daffodil.util.MaybeULong
import edu.illinois.ncsa.daffodil.util.Misc
+import edu.illinois.ncsa.daffodil.processors.charset.NBitsWidth_BitsCharsetEncoder
/**
* Helper class for creating example data that is unaligned.
@@ -75,8 +76,8 @@ object Bitte {
bb.flip()
val res = (0 to bb.limit() - 1).map { bb.get(_) }
// val bitsAsString = Misc.bytes2Bits(res.toArray)
- val enc = encoder.asInstanceOf[NonByteSizeCharsetEncoder]
- val nBits = s.length * enc.bitWidthOfACodeUnit
+ val enc = encoder.asInstanceOf[NBitsWidth_BitsCharsetEncoder]
+ val nBits = s.length * enc.bitsCharset.bitWidthOfACodeUnit
val bitStrings = res.map { b => (b & 0xFF).toBinaryString.reverse.padTo(8, '0').reverse }.toList
val allBits = bitStrings.reverse.mkString.takeRight(nBits)
val sevenBitChunks = allBits.reverse.sliding(7, 7).map { _.reverse }.toList.reverse.mkString
diff --git a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestDecoder.scala b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestDecoder.scala
index 1607cf2..d0551ee 100644
--- a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestDecoder.scala
+++ b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestDecoder.scala
@@ -35,7 +35,7 @@
//
package edu.illinois.ncsa.daffodil.io
//
-import java.nio.charset.Charset
+import java.nio.charset.{ Charset => JavaCharset }
import java.nio.charset.CodingErrorAction
import java.nio.ByteBuffer
import java.nio.CharBuffer
@@ -81,7 +81,7 @@ class TestDecoder {
* checked for enough room for a surrogate pair.
*/
@Test def testDecoder1 {
- val originalDecoder = Charset.forName("utf-8").newDecoder()
+ val originalDecoder = JavaCharset.forName("utf-8").newDecoder()
originalDecoder.onMalformedInput(CodingErrorAction.REPORT)
originalDecoder.onUnmappableCharacter(CodingErrorAction.REPORT)
val decoder = originalDecoder
@@ -148,7 +148,7 @@ class TestDecoder {
}
@Test def testDecoderWorkaround1 {
- val originalDecoder = Charset.forName("utf-8").newDecoder()
+ val originalDecoder = JavaCharset.forName("utf-8").newDecoder()
originalDecoder.onMalformedInput(CodingErrorAction.REPORT)
originalDecoder.onUnmappableCharacter(CodingErrorAction.REPORT)
val decoder = originalDecoder
diff --git a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetDecoders1Bit.scala b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetDecoders1Bit.scala
index 02e32b2..8ad61f9 100644
--- a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetDecoders1Bit.scala
+++ b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetDecoders1Bit.scala
@@ -41,6 +41,7 @@ import org.junit.Test
import edu.illinois.ncsa.daffodil.processors.charset.CharsetUtils
import edu.illinois.ncsa.daffodil.util.Misc
import edu.illinois.ncsa.daffodil.util.MaybeULong
+import edu.illinois.ncsa.daffodil.processors.charset.NBitsWidth_BitsCharsetDecoder
class TestNonByteSizedCharsetDecoders1Bit {
@@ -59,10 +60,7 @@ class TestNonByteSizedCharsetDecoders1Bit {
@Test def test1BitMSBF_02(): Unit = {
val cs1Bit = CharsetUtils.getCharset("X-DFDL-BITS-MSBF")
- val decoder = cs1Bit.newDecoder() match {
- case nbs: NonByteSizeCharsetDecoder => nbs
- case _ => null
- }
+ val decoder = cs1Bit.newDecoder().asInstanceOf[NBitsWidth_BitsCharsetDecoder]
assertNotNull(decoder)
val cb = CharBuffer.allocate(32)
val bb = ByteBuffer.wrap(Misc.hex2Bytes("DEADBEEF57"))
@@ -77,11 +75,7 @@ class TestNonByteSizedCharsetDecoders1Bit {
@Test def test1BitMSBF_03(): Unit = {
val cs1Bit = CharsetUtils.getCharset("X-DFDL-BITS-MSBF")
- val decoder = cs1Bit.newDecoder() match {
- case nbs: NonByteSizeCharsetDecoder => nbs
- case _ => null
- }
- assertNotNull(decoder)
+ val decoder = cs1Bit.newDecoder()
val cb = CharBuffer.allocate(4)
val bb = ByteBuffer.wrap(Misc.hex2Bytes("57"))
bb.limit(bb.limit - 1)
@@ -96,11 +90,7 @@ class TestNonByteSizedCharsetDecoders1Bit {
@Test def test1BitMSBF_04(): Unit = {
val cs1Bit = CharsetUtils.getCharset("X-DFDL-BITS-MSBF")
- val decoder = cs1Bit.newDecoder() match {
- case nbs: NonByteSizeCharsetDecoder => nbs
- case _ => null
- }
- assertNotNull(decoder)
+ val decoder = cs1Bit.newDecoder()
val cb = CharBuffer.allocate(40)
val bb = ByteBuffer.wrap(Misc.hex2Bytes("DEADBEEF57"))
bb.limit(bb.limit - 1)
@@ -130,10 +120,7 @@ class TestNonByteSizedCharsetDecoders1Bit {
@Test def test1BitLSBF_02(): Unit = {
val cs1Bit = CharsetUtils.getCharset("X-DFDL-BITS-LSBF")
- val decoder = cs1Bit.newDecoder() match {
- case nbs: NonByteSizeCharsetDecoder => nbs
- case _ => null
- }
+ val decoder = cs1Bit.newDecoder()
assertNotNull(decoder)
val cb = CharBuffer.allocate(32)
val bb = ByteBuffer.wrap(Misc.hex2Bytes("DEADBEEF57"))
@@ -154,11 +141,7 @@ class TestNonByteSizedCharsetDecoders1Bit {
@Test def test1BitLSBF_03(): Unit = {
val cs1Bit = CharsetUtils.getCharset("X-DFDL-BITS-LSBF")
- val decoder = cs1Bit.newDecoder() match {
- case nbs: NonByteSizeCharsetDecoder => nbs
- case _ => null
- }
- assertNotNull(decoder)
+ val decoder = cs1Bit.newDecoder()
val cb = CharBuffer.allocate(4)
val bb = ByteBuffer.wrap(Misc.hex2Bytes("57"))
bb.limit(bb.limit - 1)
@@ -173,11 +156,7 @@ class TestNonByteSizedCharsetDecoders1Bit {
@Test def test1BitLSBF_04(): Unit = {
val cs1Bit = CharsetUtils.getCharset("X-DFDL-BITS-LSBF")
- val decoder = cs1Bit.newDecoder() match {
- case nbs: NonByteSizeCharsetDecoder => nbs
- case _ => null
- }
- assertNotNull(decoder)
+ val decoder = cs1Bit.newDecoder()
val cb = CharBuffer.allocate(40)
val bb = ByteBuffer.wrap(Misc.hex2Bytes("DEADBEEF57"))
bb.limit(bb.limit - 1)
diff --git a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetDecoders3Bit.scala b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetDecoders3Bit.scala
index 374de28..a555ef4 100644
--- a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetDecoders3Bit.scala
+++ b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetDecoders3Bit.scala
@@ -58,11 +58,7 @@ class TestNonByteSizedCharsetDecoders3Bit {
@Test def test3BitMSBF_02(): Unit = {
val cs = CharsetUtils.getCharset("X-DFDL-OCTAL-MSBF")
- val decoder = cs.newDecoder() match {
- case nbs: NonByteSizeCharsetDecoder => nbs
- case _ => null
- }
- assertNotNull(decoder)
+ val decoder = cs.newDecoder()
val cb = CharBuffer.allocate(7) // not enough space for last digit
val bb = ByteBuffer.wrap(Misc.bits2Bytes("1111 000 001 010 011 100 101 110 111"))
decoder.setInitialBitOffset(4)
@@ -75,11 +71,7 @@ class TestNonByteSizedCharsetDecoders3Bit {
@Test def test3BitMSBF_03(): Unit = {
val cs = CharsetUtils.getCharset("X-DFDL-OCTAL-MSBF")
- val decoder = cs.newDecoder() match {
- case nbs: NonByteSizeCharsetDecoder => nbs
- case _ => null
- }
- assertNotNull(decoder)
+ val decoder = cs.newDecoder()
val cb = CharBuffer.allocate(4)
val bb = ByteBuffer.wrap(Misc.bits2Bytes("101 010 11"))
bb.limit(bb.limit - 1)
@@ -93,11 +85,7 @@ class TestNonByteSizedCharsetDecoders3Bit {
@Test def test3BitMSBF_04(): Unit = {
val cs = CharsetUtils.getCharset("X-DFDL-OCTAL-MSBF")
- val decoder = cs.newDecoder() match {
- case nbs: NonByteSizeCharsetDecoder => nbs
- case _ => null
- }
- assertNotNull(decoder)
+ val decoder = cs.newDecoder()
val cb = CharBuffer.allocate(40)
val bb = ByteBuffer.wrap(Misc.bits2Bytes("1111 000 0|01 010 011| 100 101 11|0 111 1111"))
bb.limit(bb.limit - 1)
@@ -124,11 +112,7 @@ class TestNonByteSizedCharsetDecoders3Bit {
@Test def test3BitLSBF_02(): Unit = {
val cs = CharsetUtils.getCharset("X-DFDL-OCTAL-LSBF")
- val decoder = cs.newDecoder() match {
- case nbs: NonByteSizeCharsetDecoder => nbs
- case _ => null
- }
- assertNotNull(decoder)
+ val decoder = cs.newDecoder()
val cb = CharBuffer.allocate(8)
val bb = ByteBuffer.wrap(Misc.bits2Bytes("1 000 1111 | 011 010 00 | 10 101 100 | 0000 111 1"))
decoder.setInitialBitOffset(4)
@@ -141,11 +125,7 @@ class TestNonByteSizedCharsetDecoders3Bit {
@Test def test3BitLSBF_03(): Unit = {
val cs = CharsetUtils.getCharset("X-DFDL-OCTAL-LSBF")
- val decoder = cs.newDecoder() match {
- case nbs: NonByteSizeCharsetDecoder => nbs
- case _ => null
- }
- assertNotNull(decoder)
+ val decoder = cs.newDecoder()
val cb = CharBuffer.allocate(4)
val bb = ByteBuffer.wrap(Misc.bits2Bytes("00 111 101"))
bb.limit(bb.limit - 1)
@@ -159,11 +139,7 @@ class TestNonByteSizedCharsetDecoders3Bit {
@Test def test3BitLSBF_04(): Unit = {
val cs = CharsetUtils.getCharset("X-DFDL-OCTAL-LSBF")
- val decoder = cs.newDecoder() match {
- case nbs: NonByteSizeCharsetDecoder => nbs
- case _ => null
- }
- assertNotNull(decoder)
+ val decoder = cs.newDecoder()
val cb = CharBuffer.allocate(40)
val bb = ByteBuffer.wrap(Misc.bits2Bytes("1 000 1111 | 011 010 00 | 10 101 100 | 0000 111 1"))
bb.limit(bb.limit - 1)
diff --git a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetEncoders1Bit.scala b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetEncoders1Bit.scala
index 9845c22..66f44c4 100644
--- a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetEncoders1Bit.scala
+++ b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetEncoders1Bit.scala
@@ -58,11 +58,7 @@ class TestNonByteSizedCharsetEncoders1Bit {
@Test def test1BitMSBF_02(): Unit = {
val cs1Bit = CharsetUtils.getCharset("X-DFDL-BITS-MSBF")
- val encoder = cs1Bit.newEncoder() match {
- case nbs: NonByteSizeCharsetEncoder => nbs
- case _ => null
- }
- assertNotNull(encoder)
+ val encoder = cs1Bit.newEncoder()
val cb = CharBuffer.wrap(Misc.hex2Bits("DEADBEEF57"))
val bb = ByteBuffer.allocate(4)
val expectedBytes = Misc.hex2Bytes("DEADBEEF").toList
@@ -88,11 +84,7 @@ class TestNonByteSizedCharsetEncoders1Bit {
@Test def test1BitLSBF_02(): Unit = {
val cs1Bit = CharsetUtils.getCharset("X-DFDL-BITS-LSBF")
- val encoder = cs1Bit.newEncoder() match {
- case nbs: NonByteSizeCharsetEncoder => nbs
- case _ => null
- }
- assertNotNull(encoder)
+ val encoder = cs1Bit.newEncoder()
val cb = CharBuffer.wrap(Misc.hex2Bits("7BB57DF757"))
val bb = ByteBuffer.allocate(4)
val expectedBytes = Misc.hex2Bytes("DEADBEEF").toList
diff --git a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetEncoders3Bit.scala b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetEncoders3Bit.scala
index e25eec0..c0c5bbc 100644
--- a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetEncoders3Bit.scala
+++ b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/io/TestNonByteSizedCharsetEncoders3Bit.scala
@@ -58,11 +58,7 @@ class TestNonByteSizedCharsetEncoders3Bit {
@Test def test3BitMSBF_02(): Unit = {
val cs = CharsetUtils.getCharset("X-DFDL-OCTAL-MSBF")
- val encoder = cs.newEncoder() match {
- case nbs: NonByteSizeCharsetEncoder => nbs
- case _ => null
- }
- assertNotNull(encoder)
+ val encoder = cs.newEncoder()
val cb = CharBuffer.wrap("012345677")
val expectedBytes = Misc.bits2Bytes("000 001 010 011 100 101 110 111").toList
val bb = ByteBuffer.allocate(3) // not enough space for last digit
@@ -88,10 +84,7 @@ class TestNonByteSizedCharsetEncoders3Bit {
@Test def test3BitLSBF_02(): Unit = {
val cs = CharsetUtils.getCharset("X-DFDL-OCTAL-LSBF")
- val encoder = cs.newEncoder() match {
- case nbs: NonByteSizeCharsetEncoder => nbs
- case _ => null
- }
+ val encoder = cs.newEncoder()
assertNotNull(encoder)
val cb = CharBuffer.wrap("012345677")
val expectedBytes = Misc.bits2Bytes("10 001 000 1 100 011 0 111 110 10").toList
diff --git a/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/processors/charset/TestBitsCharset.scala b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/processors/charset/TestBitsCharset.scala
new file mode 100644
index 0000000..a34dcba
--- /dev/null
+++ b/daffodil-io/src/test/scala/edu/illinois/ncsa/daffodil/processors/charset/TestBitsCharset.scala
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.illinois.ncsa.daffodil.processors.charset
+
+import java.nio.charset.CodingErrorAction
+import java.nio.ByteBuffer
+import java.nio.CharBuffer
+import java.nio.charset.CoderResult
+import org.junit.Test
+import org.junit.Assert._
+
+class TestBitsCharset {
+
+ @Test def testDecoder0 {
+ val jbcs = new BitsCharsetWrappingJavaCharset("utf-8")
+ val decoder = jbcs.newDecoder()
+ decoder.onMalformedInput(CodingErrorAction.REPORT)
+ decoder.onUnmappableCharacter(CodingErrorAction.REPORT)
+ val bytes = Seq(0x31, 0x32, 0x33, 0x34, 0x35).map { _.toByte }
+ val bb = ByteBuffer.wrap(bytes.toArray)
+ val cb = CharBuffer.allocate(5)
+ cb.limit(4)
+
+ val decodeCR = decoder.decode(bb, cb, true)
+ //
+ // We get an OVERFLOW
+ //
+ assertEquals(CoderResult.OVERFLOW, decodeCR)
+ assertEquals(4, cb.position())
+ assertEquals(4, bb.position())
+
+ assertEquals("1234", cb.flip().toString())
+ }
+
+ @Test def testDecoder1 {
+ val jbcs = new BitsCharsetWrappingJavaCharset("utf-8")
+ val decoder = jbcs.newDecoder()
+ decoder.onMalformedInput(CodingErrorAction.REPORT)
+ decoder.onUnmappableCharacter(CodingErrorAction.REPORT)
+ //
+ // This byte sequence "needs" the fourth byte. The first 3 bytes are consistent
+ // with the 4-byte utf-8 bytes. However, the last byte here is not a valid
+ // last byte of a 4-byte representation.
+ //
+ val bytes = Seq(0xf0, 0x90, 0x87, 0x67).map { _.toByte }
+ val bb = ByteBuffer.wrap(bytes.toArray)
+ val cb = CharBuffer.allocate(2)
+ cb.limit(1) // room for only 1 character
+
+ var decodeCR = decoder.decode(bb, cb, true)
+ //
+ // We get an OVERFLOW with nothing consumed or produced
+ //
+ assertEquals(CoderResult.OVERFLOW, decodeCR)
+ assertEquals(0, cb.position())
+ assertEquals(0, bb.position())
+ //
+ // Note that even though we passed 'true' meaning no more data possible,
+ // running out of data wasn't the issue.
+ //
+ // It appears Java's decoder for utf-8 is deciding that it WILL need
+ // to create two characters (a surrogate pair) becuase that first utf-8 byte of 0xF0
+ // indicates 4 bytes.
+ //
+ // At that point it checks if there is room for the two surrogate code units,
+ // finds that there is not, and returns OVERFLOW.
+ //
+ // Note: It isn't legal for a utf-8 decoder to return only 1 half of a
+ // surrogate pair, because what bytes would it indicate were consumed in that case?
+ // It can't say zero, because then there is no forward progress. It can't say
+ // any of them were consumed, because then they won't be there to decode
+ // in order to figure out the other half of the surrogate pair.
+ //
+ // It *could* have decided to check if the 4-bytes actually decoded correctly or not
+ // and then even though there was only room for 1 half of the surrogate pair, then
+ // issued the Malformed[3]. It doesn't do that though.
+ //
+
+ var flushCR = decoder.flush(cb) // causes IllegalStateException
+
+ //
+ // We don't even get the error here
+ //
+ // assertEquals(CoderResult.UNDERFLOW, flushCR)
+
+ //
+ // Start over
+ //
+ decoder.reset()
+ cb.limit(2) // room for 2 characters
+ decodeCR = decoder.decode(bb, cb, true)
+ //
+ // now it detects an error.
+ //
+ assertTrue(decodeCR.isMalformed())
+ assertEquals(3, decodeCR.length())
+ assertEquals(0, cb.position)
+ assertEquals(0, bb.position)
+ //
+ flushCR = decoder.flush(cb)
+ assertEquals(CoderResult.UNDERFLOW, flushCR)
+ assertEquals(0, cb.position)
+ assertEquals(0, bb.position)
+ }
+
+ @Test def testDecoderWorkaround1 {
+ val jbcs = new BitsCharsetWrappingJavaCharset("utf-8")
+ val decoder = jbcs.newDecoder()
+ decoder.onMalformedInput(CodingErrorAction.REPORT)
+ decoder.onUnmappableCharacter(CodingErrorAction.REPORT)
+ //
+ // That initial 0xF0 tells the UTF-8 decoder that
+ // it should expect a 4-byte character. That means it
+ // is going to try to create a surrogate pair (if it
+ // decodes correctly.)
+ //
+ val bytes = Seq(0xf0, 0x90, 0x87, 0x67).map { _.toByte }
+ val bb = ByteBuffer.wrap(bytes.toArray)
+ val cb = CharBuffer.allocate(2)
+ cb.limit(1) // room for only 1 character
+ var decodeCR = decoder.decode(bb, cb, true)
+ //
+ // result is strange, OVERFLOW with nothing consumed or produced
+ //
+ assertEquals(CoderResult.OVERFLOW, decodeCR)
+ assertEquals(0, cb.position())
+ assertEquals(0, bb.position())
+ //
+ //
+ val tempCB = CharBuffer.allocate(2)
+ decodeCR = decoder.decode(bb, tempCB, true)
+
+ assertTrue(decodeCR.isError)
+ assertEquals(0, tempCB.position)
+ assertEquals(0, bb.position)
+
+ val flushCR = decoder.flush(cb)
+ assertEquals(0, tempCB.position)
+ assertEquals(0, bb.position)
+
+ assertEquals(CoderResult.UNDERFLOW, flushCR)
+ //
+ // now it detects an error.
+ //
+ assertTrue(decodeCR.isMalformed())
+ assertEquals(3, decodeCR.length())
+ assertEquals(0, cb.position)
+ assertEquals(0, bb.position)
+ //
+ }
+
+}
diff --git a/daffodil-lib/src/main/scala/edu/illinois/ncsa/daffodil/util/Misc.scala b/daffodil-lib/src/main/scala/edu/illinois/ncsa/daffodil/util/Misc.scala
index 84fa12fe..435dd00 100644
--- a/daffodil-lib/src/main/scala/edu/illinois/ncsa/daffodil/util/Misc.scala
+++ b/daffodil-lib/src/main/scala/edu/illinois/ncsa/daffodil/util/Misc.scala
@@ -43,7 +43,7 @@ import java.io.ByteArrayOutputStream
import java.net.URLClassLoader
import java.nio.ByteBuffer
import java.nio.CharBuffer
-import java.nio.charset.Charset
+import java.nio.charset.{ Charset => JavaCharset }
import java.nio.charset.CodingErrorAction
import scala.collection.JavaConversions._
import edu.illinois.ncsa.daffodil.equality._
@@ -386,7 +386,7 @@ object Misc {
code.toChar
}
- private val bytesCharset = Charset.forName("windows-1252") // same as iso-8859-1 but has a few more glyphs.
+ private val bytesCharset = JavaCharset.forName("windows-1252") // same as iso-8859-1 but has a few more glyphs.
private val bytesDecoder = {
val decoder = bytesCharset.newDecoder()
decoder.onMalformedInput(CodingErrorAction.REPLACE)
@@ -492,9 +492,9 @@ object Misc {
* True if this charset encoding is suitable for display using the
* all-visible-glyph stuff above.
*/
- def isAsciiBased(csName: String): Boolean = isAsciiBased(Charset.forName(csName))
+ def isAsciiBased(csName: String): Boolean = isAsciiBased(JavaCharset.forName(csName))
- def isAsciiBased(cs: Charset): Boolean = {
+ def isAsciiBased(cs: JavaCharset): Boolean = {
val aliases: Seq[String] = cs.aliases().toSeq.map { _.toUpperCase }
val byName =
aliases.exists { s =>
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/BCDUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/BCDUnparsers.scala
index 5c09a82..8457fb3 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/BCDUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/BCDUnparsers.scala
@@ -45,7 +45,7 @@ class BCDIntegerRuntimeLengthUnparser(
extends BCDIntegerBaseUnparser(e)
with HasRuntimeExplicitLength {
- override val runtimeDependencies = List(lengthEv)
+ override lazy val runtimeDependencies = List(lengthEv)
}
final class BCDIntegerMinLengthInBytesUnparser(
@@ -55,7 +55,7 @@ final class BCDIntegerMinLengthInBytesUnparser(
override def getBitLength(state: ParseOrUnparseState): Int = {
val len = state.currentNode.get.asSimple.dataValue.asInstanceOf[Array[Byte]].length * 8
- val min = minLengthInBytes * 8
+ val min = minLengthInBytes * 8
scala.math.max(len, min)
}
}
@@ -84,7 +84,7 @@ class BCDDecimalRuntimeLengthUnparser(
extends BCDDecimalBaseUnparser(e, binaryDecimalVirtualPoint)
with HasRuntimeExplicitLength {
- override val runtimeDependencies = List(lengthEv)
+ override lazy val runtimeDependencies = List(lengthEv)
}
final class BCDDecimalMinLengthInBytesUnparser(
@@ -95,7 +95,7 @@ final class BCDDecimalMinLengthInBytesUnparser(
override def getBitLength(state: ParseOrUnparseState): Int = {
val len = state.currentNode.get.asSimple.dataValue.asInstanceOf[Array[Byte]].length * 8
- val min = minLengthInBytes * 8
+ val min = minLengthInBytes * 8
scala.math.max(len, min)
}
}
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/BinaryNumberUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/BinaryNumberUnparsers.scala
index e3c6335..726205f 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/BinaryNumberUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/BinaryNumberUnparsers.scala
@@ -46,8 +46,8 @@ import edu.illinois.ncsa.daffodil.io.DataOutputStream
import edu.illinois.ncsa.daffodil.util.Numbers._
import edu.illinois.ncsa.daffodil.io.FormatInfo
-abstract class BinaryNumberBaseUnparser(e: ElementRuntimeData)
- extends PrimUnparserObject(e) {
+abstract class BinaryNumberBaseUnparser(override val context: ElementRuntimeData)
+ extends PrimUnparser {
protected def getBitLength(s: ParseOrUnparseState): Int
@@ -69,7 +69,7 @@ abstract class BinaryNumberBaseUnparser(e: ElementRuntimeData)
if (!res) {
Assert.invariant(dos.maybeRelBitLimit0b.isDefined)
UnparseError(One(state.schemaFileLocation), One(state.currentLocation), "Insufficient space to unparse element %s, required %s bits, but only %s were available.",
- e.dpathElementCompileInfo.namedQName.toPrettyString, nBits, dos.maybeRelBitLimit0b.get)
+ context.dpathElementCompileInfo.namedQName.toPrettyString, nBits, dos.maybeRelBitLimit0b.get)
}
}
@@ -90,6 +90,9 @@ abstract class BinaryIntegerBaseUnparser(e: ElementRuntimeData, signed: Boolean)
class BinaryIntegerKnownLengthUnparser(e: ElementRuntimeData, signed: Boolean, override val lengthInBits: Int)
extends BinaryIntegerBaseUnparser(e, signed)
with HasKnownLengthInBits {
+
+ override lazy val runtimeDependencies = Nil
+
}
class BinaryIntegerRuntimeLengthUnparser(val e: ElementRuntimeData, signed: Boolean, val lengthEv: Evaluatable[JLong], val lUnits: LengthUnits)
@@ -102,9 +105,11 @@ class BinaryIntegerRuntimeLengthUnparser(val e: ElementRuntimeData, signed: Bool
class BinaryFloatUnparser(e: ElementRuntimeData)
extends BinaryNumberBaseUnparser(e) {
+ override lazy val runtimeDependencies = Nil
+
override def getBitLength(s: ParseOrUnparseState) = 32
- override def putNumber(dos: DataOutputStream, value: JNumber, nBits: Int, finfo:FormatInfo): Boolean = {
+ override def putNumber(dos: DataOutputStream, value: JNumber, nBits: Int, finfo: FormatInfo): Boolean = {
dos.putBinaryFloat(asFloat(value), finfo)
}
@@ -113,6 +118,8 @@ class BinaryFloatUnparser(e: ElementRuntimeData)
class BinaryDoubleUnparser(e: ElementRuntimeData)
extends BinaryNumberBaseUnparser(e) {
+ override lazy val runtimeDependencies = Nil
+
override def getBitLength(s: ParseOrUnparseState) = 64
override def putNumber(dos: DataOutputStream, value: JNumber, nBits: Int, finfo: FormatInfo): Boolean = {
@@ -123,6 +130,9 @@ class BinaryDoubleUnparser(e: ElementRuntimeData)
class BinaryDecimalKnownLengthUnparser(e: ElementRuntimeData, signed: YesNo, binaryDecimalVirtualPoint: Int, val lengthInBits: Int)
extends BinaryDecimalUnparserBase(e, signed, binaryDecimalVirtualPoint)
with HasKnownLengthInBits {
+
+ override lazy val runtimeDependencies = Nil
+
}
class BinaryDecimalRuntimeLengthUnparser(val e: ElementRuntimeData, signed: YesNo, binaryDecimalVirtualPoint: Int, val lengthEv: Evaluatable[JLong], val lUnits: LengthUnits)
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ConvertTextNumberUnparser.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ConvertTextNumberUnparser.scala
index f7f99fd..42011c7 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ConvertTextNumberUnparser.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ConvertTextNumberUnparser.scala
@@ -44,7 +44,9 @@ case class ConvertTextCombinatorUnparser(
rd: TermRuntimeData,
valueUnparser: Unparser,
converterUnparser: Unparser)
- extends UnparserObject(rd) {
+ extends CombinatorUnparser(rd) {
+
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Seq(converterUnparser, valueUnparser)
@@ -60,10 +62,12 @@ case class ConvertTextCombinatorUnparser(
case class ConvertTextNumberUnparser[S](
helper: ConvertTextNumberParserUnparserHelperBase[S],
nff: NumberFormatFactoryBase[S],
- erd: ElementRuntimeData)
- extends TermUnparser(erd)
+ override val context: ElementRuntimeData)
+ extends PrimUnparser
with ToBriefXMLImpl {
+ override lazy val runtimeDependencies = Nil
+
override def toString = "to(xs:" + helper.xsdType + ")"
override lazy val childProcessors = Nil
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/DelimitedUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/DelimitedUnparsers.scala
index 449adb9..7b47c46 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/DelimitedUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/DelimitedUnparsers.scala
@@ -48,13 +48,15 @@ import edu.illinois.ncsa.daffodil.util.Maybe
import edu.illinois.ncsa.daffodil.util.Maybe.Nope
import edu.illinois.ncsa.daffodil.util.Maybe.One
-sealed class StringDelimitedUnparser(erd: ElementRuntimeData,
+sealed class StringDelimitedUnparser(override val context: ElementRuntimeData,
escapeScheme: Maybe[EscapeSchemeUnparseEv],
isDelimRequired: Boolean)
- extends TextPrimUnparserObject(erd) {
+ extends TextPrimUnparser {
+
+ override lazy val runtimeDependencies = Nil
val fieldDFA = CreateFieldDFA()
- val textUnparser = new TextDelimitedUnparser(erd)
+ val textUnparser = new TextDelimitedUnparser(context)
protected def theString(state: UState) =
state.currentInfosetNode.asSimple.dataValueAsString
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/DelimiterUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/DelimiterUnparsers.scala
index a957ec4..3e6b5cf 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/DelimiterUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/DelimiterUnparsers.scala
@@ -41,8 +41,12 @@ import java.nio.charset.UnmappableCharacterException
import edu.illinois.ncsa.daffodil.util.Misc
import edu.illinois.ncsa.daffodil.exceptions.Assert
-class DelimiterTextUnparser(erd: TermRuntimeData, delimiterType: DelimiterTextType.Type)
- extends TextPrimUnparserObject(erd) {
+class DelimiterTextUnparser(override val context: TermRuntimeData, delimiterType: DelimiterTextType.Type)
+ extends TextPrimUnparser {
+
+ private def erd = context
+
+ override lazy val runtimeDependencies = Nil
override lazy val nom = {
if (delimiterType == DelimiterTextType.Initiator) "InitiatorUnparser"
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ElementKindUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ElementKindUnparsers.scala
index 39db1b9..b4fb6af 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ElementKindUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ElementKindUnparsers.scala
@@ -43,7 +43,9 @@ import edu.illinois.ncsa.daffodil.exceptions.Assert
import edu.illinois.ncsa.daffodil.equality._
class ComplexTypeUnparser(rd: RuntimeData, bodyUnparser: Unparser)
- extends UnparserObject(rd) {
+ extends CombinatorUnparser(rd) {
+
+ override lazy val runtimeDependencies = Nil
override def nom = "ComplexType"
@@ -58,9 +60,12 @@ class ComplexTypeUnparser(rd: RuntimeData, bodyUnparser: Unparser)
}
}
-class SequenceCombinatorUnparser(rdArg: ModelGroupRuntimeData, childUnparsers: Vector[Unparser])
- extends TermUnparser(rdArg)
+class SequenceCombinatorUnparser(ctxt: ModelGroupRuntimeData, childUnparsers: Vector[Unparser])
+ extends CombinatorUnparser(ctxt)
with ToBriefXMLImpl {
+
+ override lazy val runtimeDependencies = Nil
+
override def nom = "Sequence"
// Sequences of nothing (no initiator, no terminator, nothing at all) should
@@ -70,7 +75,7 @@ class SequenceCombinatorUnparser(rdArg: ModelGroupRuntimeData, childUnparsers: V
// Since some of the grammar terms might have folded away to EmptyGram,
// the number of unparsers here may be different from the number of
// children of the sequence group.
- Assert.invariant(rdArg.groupMembers.length >= childUnparsers.length)
+ Assert.invariant(ctxt.groupMembers.length >= childUnparsers.length)
override lazy val childProcessors: Seq[Processor] = childUnparsers
@@ -102,7 +107,7 @@ class SequenceCombinatorUnparser(rdArg: ModelGroupRuntimeData, childUnparsers: V
val c = ev.asComplex
//ok. We've peeked ahead and found the end of the complex element
//that this sequence is the model group of.
- val optParentRD = termRuntimeData.immediateEnclosingElementRuntimeData
+ val optParentRD = ctxt.immediateEnclosingElementRuntimeData
optParentRD match {
case Some(e: ElementRuntimeData) =>
Assert.invariant(c.runtimeData.namedQName =:= e.namedQName)
@@ -139,10 +144,12 @@ class SequenceCombinatorUnparser(rdArg: ModelGroupRuntimeData, childUnparsers: V
}
class ChoiceCombinatorUnparser(mgrd: ModelGroupRuntimeData, eventUnparserMap: Map[ChoiceBranchEvent, Unparser])
- extends TermUnparser(mgrd)
+ extends CombinatorUnparser(mgrd)
with ToBriefXMLImpl {
override def nom = "Choice"
+ override lazy val runtimeDependencies = Nil
+
override lazy val childProcessors: Seq[Processor] = eventUnparserMap.map { case (k, v) => v }.toSeq
def unparse(state: UState): Unit = {
@@ -175,9 +182,10 @@ class ChoiceCombinatorUnparser(mgrd: ModelGroupRuntimeData, eventUnparserMap: Ma
// statically at compile time. That logic is in ChoiceCombinator, and the
// branch to take is passed into this HiddenChoiceCombinatorUnparser.
class HiddenChoiceCombinatorUnparser(mgrd: ModelGroupRuntimeData, branchUnparser: Unparser)
- extends TermUnparser(mgrd)
+ extends CombinatorUnparser(mgrd)
with ToBriefXMLImpl {
override def nom = "HiddenChoice"
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors: Seq[Processor] = Seq(branchUnparser)
@@ -189,9 +197,9 @@ class HiddenChoiceCombinatorUnparser(mgrd: ModelGroupRuntimeData, branchUnparser
class DelimiterStackUnparser(initiatorOpt: Maybe[InitiatorUnparseEv],
separatorOpt: Maybe[SeparatorUnparseEv],
terminatorOpt: Maybe[TerminatorUnparseEv],
- override val context: TermRuntimeData,
+ ctxt: TermRuntimeData,
bodyUnparser: Unparser)
- extends Unparser {
+ extends CombinatorUnparser(ctxt) {
override def nom = "DelimiterStack"
override def toBriefXML(depthLimit: Int = -1): String = {
@@ -223,8 +231,8 @@ class DelimiterStackUnparser(initiatorOpt: Maybe[InitiatorUnparseEv],
}
}
-class DynamicEscapeSchemeUnparser(escapeScheme: EscapeSchemeUnparseEv, override val context: TermRuntimeData, bodyUnparser: Unparser)
- extends Unparser {
+class DynamicEscapeSchemeUnparser(escapeScheme: EscapeSchemeUnparseEv, ctxt: TermRuntimeData, bodyUnparser: Unparser)
+ extends CombinatorUnparser(ctxt) {
override def nom = "EscapeSchemeStack"
override lazy val childProcessors: Seq[Processor] = Seq(bodyUnparser)
@@ -248,8 +256,9 @@ class DynamicEscapeSchemeUnparser(escapeScheme: EscapeSchemeUnparseEv, override
}
class ArrayCombinatorUnparser(erd: ElementRuntimeData, bodyUnparser: Unparser)
- extends TermUnparser(erd) {
+ extends CombinatorUnparser(erd) {
override def nom = "Array"
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Seq(bodyUnparser)
def unparse(state: UState) {
@@ -292,10 +301,13 @@ class ArrayCombinatorUnparser(erd: ElementRuntimeData, bodyUnparser: Unparser)
}
}
-class OptionalCombinatorUnparser(erd: ElementRuntimeData, bodyUnparser: Unparser) extends UnparserObject(erd) {
+class OptionalCombinatorUnparser(erd: ElementRuntimeData, bodyUnparser: Unparser)
+ extends CombinatorUnparser(erd) {
override def nom = "Optional"
override lazy val childProcessors = Seq(bodyUnparser)
+ override lazy val runtimeDependencies = Nil
+
def unparse(state: UState) {
state.arrayIndexStack.push(1L) // one-based indexing
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ElementUnparser.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ElementUnparser.scala
index feddb26..d0c2a28 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ElementUnparser.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ElementUnparser.scala
@@ -63,6 +63,8 @@ class ElementUnspecifiedLengthUnparser(
with RegularElementUnparserStartEndStrategy
with RepMoveMixin {
+ override lazy val runtimeDependencies = Nil
+
}
trait RepMoveMixin {
@@ -94,6 +96,8 @@ class ElementUnparserNoRep(
Nope)
with RegularElementUnparserStartEndStrategy {
+ override lazy val runtimeDependencies = Nil
+
/**
* Move over in the element children, but not in the group.
* This avoids separators for this IVC element.
@@ -119,6 +123,8 @@ class ElementOVCUnspecifiedLengthUnparser(
with OVCStartEndStrategy
with RepMoveMixin {
+ override lazy val runtimeDependencies = Nil
+
}
/**
@@ -134,7 +140,7 @@ sealed abstract class ElementUnparserBase(
val eBeforeUnparser: Maybe[Unparser],
val eUnparser: Maybe[Unparser],
val eAfterUnparser: Maybe[Unparser])
- extends TermUnparser(erd)
+ extends CombinatorUnparser(erd)
with RepMoveMixin
with ElementUnparserStartEndStrategy {
@@ -186,7 +192,6 @@ sealed abstract class ElementUnparserBase(
if (state.dataProc.isDefined) state.dataProc.value.startElement(state, this)
unparseBegin(state)
- UnparserBitOrderChecks.checkUnparseBitOrder(state)
captureRuntimeValuedExpressionValues(state)
@@ -374,7 +379,7 @@ sealed trait ElementUnparserStartEndStrategy {
protected def erd: ElementRuntimeData
- protected def runtimeDependencies: Seq[Evaluatable[AnyRef]]
+ def runtimeDependencies: Seq[Evaluatable[AnyRef]]
}
sealed trait RegularElementUnparserStartEndStrategy
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ExpressionEvaluatingUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ExpressionEvaluatingUnparsers.scala
index f13332b..3dc45ae 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ExpressionEvaluatingUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/ExpressionEvaluatingUnparsers.scala
@@ -57,25 +57,27 @@ final class SetVariableSuspendableExpression(
/**
* Used when unparsing to evaluate dfdl:setVariable statements.
- *
+ *
* TODO: Possible bug. This will allow expressions to forward reference, even
* when the variables are being referenced from expressions that are NOT
- * allowed to forward reference - e.g., property value expressions such
- * as delimiters and byte order.
- *
+ * allowed to forward reference - e.g., property value expressions such
+ * as delimiters and byte order.
+ *
* This forward suspension is only supposed to be allowed for dfdl:outputValueCalc.
*/
final class SetVariableUnparser(
val expr: CompiledExpression[AnyRef],
- val rd: VariableRuntimeData,
+ override val context: VariableRuntimeData,
referencingContext: NonTermRuntimeData)
- extends UnparserObject(rd) {
+ extends PrimUnparserNoData {
+
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Nil
def suspendableExpression =
new SetVariableSuspendableExpression(
- expr, rd, referencingContext)
+ expr, context, referencingContext)
override def unparse(state: UState): Unit = {
suspendableExpression.run(state)
@@ -85,26 +87,30 @@ final class SetVariableUnparser(
// When implemented this almost certainly wants to be a combinator
// Not two separate unparsers.
-class NewVariableInstanceStartUnparser(vrd: RuntimeData)
- extends UnparserObject(vrd) {
+class NewVariableInstanceStartUnparser(override val context: RuntimeData)
+ extends PrimUnparserNoData {
+
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Nil
- vrd.notYetImplemented("newVariableInstance")
+ context.notYetImplemented("newVariableInstance")
override def unparse(ustate: UState) = {
- vrd.notYetImplemented("newVariableInstance")
+ context.notYetImplemented("newVariableInstance")
}
}
-class NewVariableInstanceEndUnparser(vrd: RuntimeData)
- extends UnparserObject(vrd) {
+class NewVariableInstanceEndUnparser(override val context: RuntimeData)
+ extends PrimUnparserNoData {
+
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Nil
- vrd.notYetImplemented("newVariableInstance")
+ context.notYetImplemented("newVariableInstance")
override def unparse(ustate: UState) = {
- vrd.notYetImplemented("newVariableInstance")
+ context.notYetImplemented("newVariableInstance")
}
}
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/FramingUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/FramingUnparsers.scala
index 3602901..cdde22f 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/FramingUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/FramingUnparsers.scala
@@ -33,15 +33,14 @@
package edu.illinois.ncsa.daffodil.processors.unparsers
import edu.illinois.ncsa.daffodil.processors.SuspendableOperation
-import edu.illinois.ncsa.daffodil.processors.SuspendableUnparser
import edu.illinois.ncsa.daffodil.util.LogLevel
import edu.illinois.ncsa.daffodil.processors.TextProcessor
import edu.illinois.ncsa.daffodil.processors.TermRuntimeData
class SkipRegionUnparser(
skipInBits: Int,
- e: TermRuntimeData)
- extends PrimUnparserObject(e) {
+ override val context: TermRuntimeData)
+ extends PrimUnparser {
override def runtimeDependencies = Nil
@@ -80,15 +79,15 @@ class AlignmentFillUnparserSuspendableOperation(
class AlignmentFillUnparser(
alignmentInBits: Int,
- val rd: TermRuntimeData)
- extends PrimUnparserObject(rd)
+ override val context: TermRuntimeData)
+ extends PrimUnparser
with SuspendableUnparser {
override def runtimeDependencies = Nil
override def suspendableOperation =
new AlignmentFillUnparserSuspendableOperation(
- alignmentInBits, rd)
+ alignmentInBits, context)
}
class MandatoryTextAlignmentUnparser(
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/HexBinaryLengthUnparser.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/HexBinaryLengthUnparser.scala
index cd3bb47..4bf83a1 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/HexBinaryLengthUnparser.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/HexBinaryLengthUnparser.scala
@@ -38,8 +38,10 @@ import edu.illinois.ncsa.daffodil.infoset.RetryableException
import edu.illinois.ncsa.daffodil.processors.UnparseTargetLengthInBitsEv
import edu.illinois.ncsa.daffodil.exceptions.Assert
-abstract class HexBinaryUnparserBase(erd: ElementRuntimeData)
- extends PrimUnparserObject(erd) {
+abstract class HexBinaryUnparserBase(override val context: ElementRuntimeData)
+ extends PrimUnparser {
+
+ override lazy val runtimeDependencies = Nil
protected def getLengthInBits(state: UState): Long
@@ -51,7 +53,7 @@ abstract class HexBinaryUnparserBase(erd: ElementRuntimeData)
val lengthInBytes = (lengthInBits + 7) / 8
if (value.length > lengthInBytes) {
- UnparseError(One(erd.schemaFileLocation), One(state.currentLocation), "Data length %d bits exceeds explicit length value: %d bits", value.length * 8, lengthInBits)
+ UnparseError(One(context.schemaFileLocation), One(state.currentLocation), "Data length %d bits exceeds explicit length value: %d bits", value.length * 8, lengthInBits)
}
val bitsFromValueToPut =
@@ -72,7 +74,7 @@ abstract class HexBinaryUnparserBase(erd: ElementRuntimeData)
if (bitsFromValueToPut > 0) {
val ret = dos.putByteArray(value, bitsFromValueToPut.toInt, state)
if (!ret) {
- UnparseError(One(erd.schemaFileLocation), One(state.currentLocation), "Failed to write %d hexBinary bits", bitsFromValueToPut)
+ UnparseError(One(context.schemaFileLocation), One(state.currentLocation), "Failed to write %d hexBinary bits", bitsFromValueToPut)
}
}
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/IBM4690PackedDecimalUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/IBM4690PackedDecimalUnparsers.scala
index 277c2d6..7f1f331 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/IBM4690PackedDecimalUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/IBM4690PackedDecimalUnparsers.scala
@@ -45,7 +45,7 @@ class IBM4690PackedIntegerRuntimeLengthUnparser(
extends IBM4690PackedIntegerBaseUnparser(e)
with HasRuntimeExplicitLength {
- override val runtimeDependencies = List(lengthEv)
+ override lazy val runtimeDependencies = List(lengthEv)
}
final class IBM4690PackedIntegerMinLengthInBytesUnparser(
@@ -84,7 +84,7 @@ class IBM4690PackedDecimalRuntimeLengthUnparser(
extends IBM4690PackedDecimalBaseUnparser(e, binaryDecimalVirtualPoint)
with HasRuntimeExplicitLength {
- override val runtimeDependencies = List(lengthEv)
+ override lazy val runtimeDependencies = List(lengthEv)
}
final class IBM4690PackedDecimalMinLengthInBytesUnparser(
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/NilEmptyCombinatorUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/NilEmptyCombinatorUnparsers.scala
index c278dab..2edad98 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/NilEmptyCombinatorUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/NilEmptyCombinatorUnparsers.scala
@@ -37,7 +37,9 @@ import edu.illinois.ncsa.daffodil.exceptions.Assert
import edu.illinois.ncsa.daffodil.util.Maybe
case class SimpleNilOrEmptyOrValueUnparser(ctxt: ElementRuntimeData,
- nilUnparser: Unparser, emptyUnparser: Unparser, valueUnparser: Unparser) extends UnparserObject(ctxt) {
+ nilUnparser: Unparser, emptyUnparser: Unparser, valueUnparser: Unparser) extends CombinatorUnparser(ctxt) {
+
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Seq(nilUnparser, emptyUnparser, valueUnparser)
@@ -58,7 +60,9 @@ case class SimpleNilOrEmptyOrValueUnparser(ctxt: ElementRuntimeData,
}
case class SimpleNilOrValueUnparser(ctxt: ElementRuntimeData,
- nilUnparser: Unparser, valueUnparser: Unparser) extends UnparserObject(ctxt) {
+ nilUnparser: Unparser, valueUnparser: Unparser) extends CombinatorUnparser(ctxt) {
+
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Seq(nilUnparser, valueUnparser)
@@ -72,7 +76,9 @@ case class SimpleNilOrValueUnparser(ctxt: ElementRuntimeData,
}
case class SimpleEmptyOrValueUnparser(ctxt: ElementRuntimeData,
- emptyUnparser: Unparser, valueUnparser: Unparser) extends UnparserObject(ctxt) {
+ emptyUnparser: Unparser, valueUnparser: Unparser) extends CombinatorUnparser(ctxt) {
+
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Seq(emptyUnparser, valueUnparser)
@@ -85,7 +91,9 @@ case class SimpleEmptyOrValueUnparser(ctxt: ElementRuntimeData,
}
case class ComplexNilOrContentUnparser(ctxt: ElementRuntimeData,
- nilUnparser: Unparser, contentUnparser: Unparser) extends UnparserObject(ctxt) {
+ nilUnparser: Unparser, contentUnparser: Unparser) extends CombinatorUnparser(ctxt) {
+
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Seq(nilUnparser, contentUnparser)
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/OptionalInfixSepUnparser.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/OptionalInfixSepUnparser.scala
index 1492f66..604fc95 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/OptionalInfixSepUnparser.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/OptionalInfixSepUnparser.scala
@@ -36,7 +36,9 @@ import edu.illinois.ncsa.daffodil.processors.TermRuntimeData
class OptionalInfixSepUnparser(contextArg: TermRuntimeData,
sepUnparser: Unparser)
- extends UnparserObject(contextArg) {
+ extends CombinatorUnparser(contextArg) {
+
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = List(sepUnparser)
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedBinaryUnparserTraits.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedBinaryUnparserTraits.scala
index e95f7ec..fbcb6ec 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedBinaryUnparserTraits.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedBinaryUnparserTraits.scala
@@ -22,21 +22,24 @@ import java.math.{ BigInteger => JBigInteger, BigDecimal => JBigDecimal }
import edu.illinois.ncsa.daffodil.exceptions.Assert
import edu.illinois.ncsa.daffodil.io.DataOutputStream
import edu.illinois.ncsa.daffodil.io.FormatInfo
+import edu.illinois.ncsa.daffodil.processors.Evaluatable
trait PackedBinaryConversion {
def fromBigInteger(bigInt: JBigInteger, nBits: Int): Array[Byte]
}
abstract class PackedBinaryBaseUnparser(
- e: ElementRuntimeData)
- extends PrimUnparserObject(e)
+ override val context: ElementRuntimeData)
+ extends PrimUnparser
with PackedBinaryConversion {
+ override lazy val runtimeDependencies: Seq[Evaluatable[AnyRef]] = Seq()
+
protected def getBitLength(s: ParseOrUnparseState): Int
def putNumber(dos: DataOutputStream, number: JNumber, nBits: Int, finfo: FormatInfo): Boolean
- def unparse(state: UState): Unit = {
+ override def unparse(state: UState): Unit = {
val nBits = getBitLength(state)
val node = state.currentInfosetNode.asSimple
val value = node.dataValue.asInstanceOf[JNumber]
@@ -52,7 +55,7 @@ abstract class PackedBinaryBaseUnparser(
if (!res) {
Assert.invariant(dos.maybeRelBitLimit0b.isDefined)
UnparseError(One(state.schemaFileLocation), One(state.currentLocation), "Insufficient space to unparse element %s, required %s bits, but only %s were available.",
- e.dpathElementCompileInfo.namedQName.toPrettyString, nBits, dos.maybeRelBitLimit0b.get)
+ context.dpathElementCompileInfo.namedQName.toPrettyString, nBits, dos.maybeRelBitLimit0b.get)
}
}
@@ -63,34 +66,33 @@ abstract class PackedBinaryDecimalBaseUnparser(
binaryDecimalVirtualPoint: Int)
extends PackedBinaryBaseUnparser(e) {
- override def putNumber(dos: DataOutputStream, number: JNumber, nBits: Int, finfo: FormatInfo): Boolean = {
+ override def putNumber(dos: DataOutputStream, number: JNumber, nBits: Int, finfo: FormatInfo): Boolean = {
- val bigDec = number.asInstanceOf[JBigDecimal]
- if (bigDec.movePointRight(binaryDecimalVirtualPoint).scale != 0)
- e.schemaDefinitionError("Decimal point of number '%s' does not match the binaryVirtualDecmialPoint: %d", bigDec, binaryDecimalVirtualPoint)
+ val bigDec = number.asInstanceOf[JBigDecimal]
+ if (bigDec.movePointRight(binaryDecimalVirtualPoint).scale != 0)
+ e.schemaDefinitionError("Decimal point of number '%s' does not match the binaryVirtualDecmialPoint: %d", bigDec, binaryDecimalVirtualPoint)
- dos.putByteArray(
- fromBigInteger(bigDec.unscaledValue, nBits),
- nBits,
- finfo)
- }
+ dos.putByteArray(
+ fromBigInteger(bigDec.unscaledValue, nBits),
+ nBits,
+ finfo)
+ }
}
-
abstract class PackedBinaryIntegerBaseUnparser(
e: ElementRuntimeData)
extends PackedBinaryBaseUnparser(e) {
- override def putNumber(dos: DataOutputStream, number: JNumber, nBits: Int, finfo: FormatInfo): Boolean = {
+ override def putNumber(dos: DataOutputStream, number: JNumber, nBits: Int, finfo: FormatInfo): Boolean = {
- val bigInt = number.isInstanceOf[JBigInteger] match {
- case true => number.asInstanceOf[JBigInteger]
- case false => new JBigInteger(number.toString)
- }
-
- dos.putByteArray(
- fromBigInteger(bigInt, nBits),
- nBits,
- finfo)
+ val bigInt = number.isInstanceOf[JBigInteger] match {
+ case true => number.asInstanceOf[JBigInteger]
+ case false => new JBigInteger(number.toString)
}
+
+ dos.putByteArray(
+ fromBigInteger(bigInt, nBits),
+ nBits,
+ finfo)
+ }
}
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedDecimalUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedDecimalUnparsers.scala
index fd429fd..e09a6b5 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedDecimalUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/PackedDecimalUnparsers.scala
@@ -48,7 +48,7 @@ class PackedIntegerRuntimeLengthUnparser(
extends PackedIntegerBaseUnparser(e, packedSignCodes)
with HasRuntimeExplicitLength {
- override val runtimeDependencies = List(lengthEv)
+ override lazy val runtimeDependencies = List(lengthEv)
}
final class PackedIntegerMinLengthInBytesUnparser(
@@ -92,7 +92,7 @@ class PackedDecimalRuntimeLengthUnparser(
extends PackedDecimalBaseUnparser(e, binaryDecimalVirtualPoint, packedSignCodes)
with HasRuntimeExplicitLength {
- override val runtimeDependencies = List(lengthEv)
+ override lazy val runtimeDependencies = List(lengthEv)
}
final class PackedDecimalMinLengthInBytesUnparser(
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/SpecifiedLength2.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/SpecifiedLength2.scala
index efdd9db..683fba6 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/SpecifiedLength2.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/SpecifiedLength2.scala
@@ -39,14 +39,13 @@ import edu.illinois.ncsa.daffodil.infoset.DISimple
import edu.illinois.ncsa.daffodil.processors.ElementRuntimeData
import edu.illinois.ncsa.daffodil.processors.SuspendableOperation
import edu.illinois.ncsa.daffodil.processors.UnparseTargetLengthInBitsEv
-import edu.illinois.ncsa.daffodil.processors.charset.DFDLCharset
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharset
import edu.illinois.ncsa.daffodil.util.LogLevel
import edu.illinois.ncsa.daffodil.util.Maybe
import edu.illinois.ncsa.daffodil.util.Maybe._
import edu.illinois.ncsa.daffodil.util.MaybeChar
import edu.illinois.ncsa.daffodil.util.MaybeULong
import edu.illinois.ncsa.daffodil.util.Misc
-import edu.illinois.ncsa.daffodil.processors.SuspendableUnparser
import edu.illinois.ncsa.daffodil.processors.CharsetEv
import edu.illinois.ncsa.daffodil.processors.LengthEv
import edu.illinois.ncsa.daffodil.processors.Evaluatable
@@ -225,7 +224,9 @@ class OVCRetryUnparser(override val context: ElementRuntimeData,
}
class CaptureStartOfContentLengthUnparser(override val context: ElementRuntimeData)
- extends PrimUnparserObject(context) {
+ extends PrimUnparser {
+
+ override lazy val runtimeDependencies = Nil
override def unparse(state: UState) {
val dos = state.dataOutputStream
@@ -239,7 +240,9 @@ class CaptureStartOfContentLengthUnparser(override val context: ElementRuntimeDa
}
class CaptureEndOfContentLengthUnparser(override val context: ElementRuntimeData, maybeFixedLengthInBits: MaybeULong)
- extends PrimUnparserObject(context) {
+ extends PrimUnparser {
+
+ override lazy val runtimeDependencies = Nil
override def unparse(state: UState) {
val dos = state.dataOutputStream.asInstanceOf[DirectOrBufferedDataOutputStream]
@@ -266,7 +269,9 @@ class CaptureEndOfContentLengthUnparser(override val context: ElementRuntimeData
}
class CaptureStartOfValueLengthUnparser(override val context: ElementRuntimeData)
- extends PrimUnparserObject(context) {
+ extends PrimUnparser {
+
+ override lazy val runtimeDependencies = Nil
override def unparse(state: UState) {
val dos = state.dataOutputStream
@@ -280,7 +285,9 @@ class CaptureStartOfValueLengthUnparser(override val context: ElementRuntimeData
}
class CaptureEndOfValueLengthUnparser(override val context: ElementRuntimeData)
- extends PrimUnparserObject(context) {
+ extends PrimUnparser {
+
+ override lazy val runtimeDependencies = Nil
override def unparse(state: UState) {
val dos = state.dataOutputStream
@@ -394,7 +401,7 @@ sealed trait NeedValueAndTargetLengthMixin {
val v = valueString(s, ustate)
val vlChars = v.length
val nPadChars = tlChars - vlChars // negative if data too long for available space.
- val cs: DFDLCharset = maybeCharsetEv.get.evaluate(ustate)
+ val cs: BitsCharset = maybeCharsetEv.get.evaluate(ustate)
val paddingLengthInBits = cs.padCharWidthInBits * nPadChars
paddingLengthInBits
}
@@ -448,19 +455,19 @@ class ElementUnusedUnparserSuspendableOperation(
}
class ElementUnusedUnparser(
- val rd: ElementRuntimeData,
+ override val context: ElementRuntimeData,
targetLengthEv: UnparseTargetLengthInBitsEv,
maybeLengthEv: Maybe[LengthEv],
maybeCharsetEv: Maybe[CharsetEv],
maybeLiteralNilEv: Maybe[NilStringLiteralForUnparserEv])
- extends PrimUnparserObject(rd)
+ extends PrimUnparser
with SuspendableUnparser {
override lazy val runtimeDependencies = List(targetLengthEv)
override def suspendableOperation =
new ElementUnusedUnparserSuspendableOperation(
- rd, targetLengthEv, maybeLengthEv, maybeCharsetEv, maybeLiteralNilEv)
+ context, targetLengthEv, maybeLengthEv, maybeCharsetEv, maybeLiteralNilEv)
}
@@ -486,7 +493,7 @@ trait PaddingUnparserMixin
protected final def charset(state: UState) =
maybeCharsetEv.get.evaluate(state)
- protected final def charWidthInBits(charset: DFDLCharset) = {
+ protected final def charWidthInBits(charset: BitsCharset) = {
val res = charset.maybeFixedWidth.get
res
}
@@ -529,20 +536,20 @@ class OnlyPaddingUnparserSuspendableOperation(override val rd: ElementRuntimeDat
* Doesn't matter if we're left or right padding if we're the only padding
*/
class OnlyPaddingUnparser(
- val rd: ElementRuntimeData,
+ override val context: ElementRuntimeData,
targetLengthEv: Evaluatable[MaybeJULong],
maybeLengthEv: Maybe[LengthEv],
maybeCharsetEv: Maybe[CharsetEv],
maybeLiteralNilEv: Maybe[NilStringLiteralForUnparserEv],
maybePadChar: MaybeChar)
- extends TextPrimUnparserObject(rd)
+ extends TextPrimUnparser
with SuspendableUnparser {
override lazy val runtimeDependencies = List(targetLengthEv)
override def suspendableOperation =
new OnlyPaddingUnparserSuspendableOperation(
- rd, targetLengthEv, maybeLengthEv, maybeCharsetEv, maybeLiteralNilEv, maybePadChar)
+ context, targetLengthEv, maybeLengthEv, maybeCharsetEv, maybeLiteralNilEv, maybePadChar)
}
class NilLiteralCharacterUnparserSuspendableOperation(override val rd: ElementRuntimeData,
@@ -580,18 +587,18 @@ class NilLiteralCharacterUnparserSuspendableOperation(override val rd: ElementRu
}
class NilLiteralCharacterUnparser(
- val rd: ElementRuntimeData,
+ override val context: ElementRuntimeData,
val targetLengthEv: UnparseTargetLengthInBitsEv,
val maybeLengthEv: Maybe[LengthEv],
val maybeCharsetEv: Maybe[CharsetEv],
literalNilChar: Char)
- extends TextPrimUnparserObject(rd)
+ extends TextPrimUnparser
with SuspendableUnparser {
override lazy val runtimeDependencies = List(targetLengthEv)
override def suspendableOperation = new NilLiteralCharacterUnparserSuspendableOperation(
- rd, targetLengthEv, maybeLengthEv, maybeCharsetEv, literalNilChar)
+ context, targetLengthEv, maybeLengthEv, maybeCharsetEv, literalNilChar)
}
@@ -639,7 +646,7 @@ class LeftCenteredPaddingUnparserSuspendableOperation(override val rd: ElementRu
}
}
-class LeftCenteredPaddingUnparser(override val rd: ElementRuntimeData,
+class LeftCenteredPaddingUnparser(rd: ElementRuntimeData,
targetLengthEv: Evaluatable[MaybeJULong],
maybeLengthEv: Maybe[LengthEv],
maybeCharsetEv: Maybe[CharsetEv],
diff --git a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/SpecifiedLengthUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/SpecifiedLengthUnparsers.scala
index 2496616..3b063f8 100644
--- a/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/SpecifiedLengthUnparsers.scala
+++ b/daffodil-runtime1-unparser/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/SpecifiedLengthUnparsers.scala
@@ -65,7 +65,9 @@ final class SpecifiedLengthExplicitImplicitUnparser(
erd: ElementRuntimeData,
targetLengthInBitsEv: UnparseTargetLengthInBitsEv,
maybeTargetLengthInCharactersEv: Maybe[UnparseTargetLengthInCharactersEv])
- extends UnparserObject(erd) {
+ extends CombinatorUnparser(erd) {
+
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Seq(eUnparser)
@@ -167,7 +169,7 @@ final class SpecifiedLengthExplicitImplicitUnparser(
//
state.schemaDefinitionUnless(erd.isSimpleType,
"Variable width character encoding '%s', dfdl:lengthKind '%s' and dfdl:lengthUnits '%s' are not supported for complex types.",
- getCharset(state).charsetName, lengthKind.toString, lengthUnits.toString)
+ getCharset(state).name, lengthKind.toString, lengthUnits.toString)
Assert.invariant(erd.isSimpleType)
Assert.invariant(this.maybeTargetLengthInCharactersEv.isDefined)
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/dsom/EncodingLattice.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/dsom/EncodingLattice.scala
index 1ac7c56..19c8050 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/dsom/EncodingLattice.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/dsom/EncodingLattice.scala
@@ -2,25 +2,25 @@
*
* Developed by: Tresys Technology, LLC
* http://www.tresys.com
- *
+ *
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal with
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
* of the Software, and to permit persons to whom the Software is furnished to do
* so, subject to the following conditions:
- *
+ *
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimers.
- *
+ *
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimers in the
* documentation and/or other materials provided with the distribution.
- *
+ *
* 3. Neither the names of Tresys Technology, nor the names of its contributors
* may be used to endorse or promote products derived from this Software
* without specific prior written permission.
- *
+ *
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/infoset/InfosetImpl.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/infoset/InfosetImpl.scala
index d5c0ccd..e9f38b7 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/infoset/InfosetImpl.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/infoset/InfosetImpl.scala
@@ -646,6 +646,11 @@ sealed trait DIElementSharedMembersMixin {
/*
* Initialized on demand
*/
+ protected final def clearContentLength() = {
+ if (_contentLength eq null) ()
+ else _contentLength.clear()
+ }
+
final def contentLength = {
if (_contentLength eq null) {
_contentLength = allocContentLength
@@ -653,6 +658,11 @@ sealed trait DIElementSharedMembersMixin {
_contentLength
}
+ protected final def clearValueLength() = {
+ if (_valueLength eq null) ()
+ else _valueLength.clear()
+ }
+
final def valueLength = {
if (_valueLength eq null) {
_valueLength = allocValueLength
@@ -694,8 +704,8 @@ sealed trait DIElementSharedImplMixin
override def clear() {
this._isNilled = false
this._validity = MaybeBoolean.Nope
- contentLength.clear()
- valueLength.clear()
+ clearContentLength()
+ clearValueLength()
}
}
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/Dynamic.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/Dynamic.scala
index bf8b47e..6b63e80 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/Dynamic.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/Dynamic.scala
@@ -114,7 +114,7 @@ trait Dynamic {
// during runtime.
def cacheConstantExpression[A <: AnyRef, B <: AnyRef](e: Evaluatable[A])(conv: (A) => B): CachedDynamic[A, B] = {
if (e.isConstant) {
- val v: A = e.optConstant.get
+ val v: A = e.maybeConstant.get
Right(conv(v))
} else {
Left(e)
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EncodingRuntimeData.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EncodingRuntimeData.scala
index 35bf367..63b4f02 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EncodingRuntimeData.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EncodingRuntimeData.scala
@@ -32,18 +32,19 @@
package edu.illinois.ncsa.daffodil.processors
-import edu.illinois.ncsa.daffodil.dsom._
+import edu.illinois.ncsa.daffodil.dsom.EncodingLattice
+import edu.illinois.ncsa.daffodil.dsom.ImplementsThrowsSDE
import edu.illinois.ncsa.daffodil.exceptions.Assert
import edu.illinois.ncsa.daffodil.exceptions.SchemaFileLocation
import edu.illinois.ncsa.daffodil.exceptions.ThrowsSDE
-import edu.illinois.ncsa.daffodil.processors.charset.DFDLCharset
-import edu.illinois.ncsa.daffodil.schema.annotation.props.gen._
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharset
+import edu.illinois.ncsa.daffodil.processors.charset.StandardBitsCharsets
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.EncodingErrorPolicy
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.UTF16Width
+import edu.illinois.ncsa.daffodil.util.Maybe
import edu.illinois.ncsa.daffodil.util.PreSerialization
import edu.illinois.ncsa.daffodil.util.TransientParam
-import java.nio.charset.StandardCharsets
-import edu.illinois.ncsa.daffodil.util.Maybe
+import edu.illinois.ncsa.daffodil.processors.charset.CharsetUtils
/**
* To eliminate circularities between RuntimeData objects and the
@@ -66,19 +67,17 @@ trait KnownEncodingMixin { self: ThrowsSDE =>
def charsetEv: CharsetEv
def knownEncodingAlignmentInBits: Int
- def optionUTF16Width: Option[UTF16Width]
-
/**
* Note that the canonical form for encoding names is all upper case.
*/
final lazy val knownEncodingName = {
Assert.invariant(isKnownEncoding)
- val res = charsetEv.optConstant.get.charsetName
+ val res = charsetEv.optConstant.get.name
res
}
final lazy val knownEncodingCharset = {
- new DFDLCharset(knownEncodingName)
+ CharsetUtils.getCharset(knownEncodingName)
}
/**
@@ -98,9 +97,9 @@ trait KnownEncodingMixin { self: ThrowsSDE =>
final lazy val knownEncodingWidthInBits = encodingMinimumCodePointWidthInBits(knownEncodingCharset)
- final def encodingMinimumCodePointWidthInBits(cs: DFDLCharset) = {
- val res = cs.charset match {
- case StandardCharsets.UTF_8 => 8
+ final def encodingMinimumCodePointWidthInBits(cs: BitsCharset) = {
+ val res = cs match {
+ case StandardBitsCharsets.UTF_8 => 8
case _ => cs.maybeFixedWidth.get
}
res
@@ -139,7 +138,6 @@ trait KnownEncodingMixin { self: ThrowsSDE =>
final class EncodingRuntimeData(
@TransientParam termRuntimeDataArg: => TermRuntimeData,
@TransientParam charsetEvArg: => CharsetEv,
- @TransientParam checkEncodingEvArg: => CheckEncodingEv,
override val schemaFileLocation: SchemaFileLocation,
optionUTF16WidthArg: Option[UTF16Width],
val defaultEncodingErrorPolicy: EncodingErrorPolicy,
@@ -149,41 +147,33 @@ final class EncodingRuntimeData(
override val knownEncodingAlignmentInBits: Int)
extends KnownEncodingMixin with ImplementsThrowsSDE with PreSerialization {
- private val optionUTF16Width_ = optionUTF16WidthArg
private val maybeUTF16Width_ = Maybe.toMaybe[UTF16Width](optionUTF16WidthArg)
- @deprecated("20170530", "Use maybeUTF16Width instead.")
- def optionUTF16Width = optionUTF16Width_
def maybeUTF16Width = maybeUTF16Width_
lazy val termRuntimeData = termRuntimeDataArg
lazy val charsetEv = charsetEvArg
- lazy val checkEncodingEv = checkEncodingEvArg
- lazy val runtimeDependencies = List(charsetEv, checkEncodingEv)
+ lazy val runtimeDependencies = List(charsetEv)
def getDecoderInfo(state: ParseOrUnparseState) = {
- checkEncodingEv.evaluate(state)
val cs = charsetEv.evaluate(state)
- val dec = state.getDecoderInfo(cs.charset)
+ val dec = state.getDecoderInfo(cs)
dec
}
def getEncoderInfo(state: ParseOrUnparseState) = {
- checkEncodingEv.evaluate(state)
val cs = charsetEv.evaluate(state)
- val enc = state.getEncoderInfo(cs.charset)
+ val enc = state.getEncoderInfo(cs)
enc
}
- def getEncoder(state: ParseOrUnparseState, dcs: DFDLCharset) = {
- checkEncodingEv.evaluate(state)
- val enc = state.getEncoder(dcs.charset)
+ def getEncoder(state: ParseOrUnparseState, cs: BitsCharset) = {
+ val enc = state.getEncoder(cs)
enc
}
- def getDFDLCharset(state: ParseOrUnparseState): DFDLCharset = {
- checkEncodingEv.evaluate(state)
+ def getDFDLCharset(state: ParseOrUnparseState): BitsCharset = {
val cs = charsetEv.evaluate(state)
cs
}
@@ -192,7 +182,6 @@ final class EncodingRuntimeData(
super.preSerialization
termRuntimeData
charsetEv
- checkEncodingEv
}
@throws(classOf[java.io.IOException])
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EvByteOrder.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EvByteOrder.scala
index cf4ccaf..856ace2 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EvByteOrder.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EvByteOrder.scala
@@ -35,7 +35,6 @@ package edu.illinois.ncsa.daffodil.processors
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen._
import edu.illinois.ncsa.daffodil.dsom._
import edu.illinois.ncsa.daffodil.equality._
-import edu.illinois.ncsa.daffodil.io.NonByteSizeCharset
/**
* Runtime valued properties that are enums would all work like ByteOrder here.
@@ -93,15 +92,24 @@ class CheckBitOrderAndCharsetEv(t: TermRuntimeData, bitOrder: BitOrder, charsetE
override lazy val runtimeDependencies = List(charsetEv)
override final protected def compute(state: ParseOrUnparseState): Ok = {
- val dfdlCS = charsetEv.evaluate(state)
- dfdlCS.charset match {
- case nbsc: NonByteSizeCharset =>
- if (nbsc.requiredBitOrder !=:= bitOrder) {
- t.schemaDefinitionError("Encoding '%s' requires bit order '%s', but bit order was '%s'.", dfdlCS.charsetName, nbsc.requiredBitOrder, bitOrder)
- }
- case _ => // do nothing
+ val bitsCharset = charsetEv.evaluate(state)
+ //
+ // If the encoding is byte aligned then bit order doesn't matter
+ // but otherwise it does.
+ //
+ // This is checking for the situation where we are within a byte,
+ // and the character encoding uses say, MSBF, but we were left
+ // off at a LSBF bit position. Or vice versa.
+ //
+ if (bitsCharset.mandatoryBitAlignment != 8
+ && (bitsCharset.requiredBitOrder !=:= bitOrder)
+ && state.bitPos1b % 8 != 1 // real runtime check
+ // we check that last because the others might fail at compile time for this Ev
+ // which would mean no possible error message at runtime, and
+ // therefore faster speed for this Ev.
+ ) {
+ t.schemaDefinitionError("Encoding '%s' requires bit order '%s', but bit order was '%s'.", bitsCharset.name, bitsCharset.requiredBitOrder, bitOrder)
}
-
Ok
}
}
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EvEncoding.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EvEncoding.scala
index 0be70e7..cd66e9b 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EvEncoding.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EvEncoding.scala
@@ -33,13 +33,14 @@
package edu.illinois.ncsa.daffodil.processors
import edu.illinois.ncsa.daffodil.dsom._
-import edu.illinois.ncsa.daffodil.processors.charset.DFDLCharset
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharset
import edu.illinois.ncsa.daffodil.processors.charset.CharsetUtils
import edu.illinois.ncsa.daffodil.exceptions.Assert
-import edu.illinois.ncsa.daffodil.io.NonByteSizeCharset
import edu.illinois.ncsa.daffodil.util.MaybeInt
import edu.illinois.ncsa.daffodil.cookers.FillByteCooker
import edu.illinois.ncsa.daffodil.cookers.EncodingCooker
+import edu.illinois.ncsa.daffodil.processors.charset.NBitsWidth_BitsCharset
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharsetWrappingJavaCharset
/*
* The way encoding works, is if a EncodingChangeParser or Unparser is
@@ -62,7 +63,7 @@ import edu.illinois.ncsa.daffodil.cookers.EncodingCooker
/**
* Encoding is a string, so there is no converter.
*/
-class EncodingEv(expr: CompiledExpression[String], trd: TermRuntimeData)
+class EncodingEv(override val expr: CompiledExpression[String], trd: TermRuntimeData)
extends EvaluatableConvertedExpression[String, String](
expr,
EncodingCooker, // cooker insures upper-case and trimmed of whitespace.
@@ -88,17 +89,29 @@ class EncodingEv(expr: CompiledExpression[String], trd: TermRuntimeData)
}
}
-class CharsetEv(encodingEv: EncodingEv, val trd: TermRuntimeData)
- extends Evaluatable[DFDLCharset](trd)
- with InfosetCachedEvaluatable[DFDLCharset] {
+final class CharsetEv(encodingEv: EncodingEv, val trd: TermRuntimeData)
+ extends Evaluatable[BitsCharset](trd)
+ with InfosetCachedEvaluatable[BitsCharset] {
override lazy val runtimeDependencies = Seq(encodingEv)
+ private def checkCharset(state: ParseOrUnparseState, bitsCharset: BitsCharset) {
+ bitsCharset match {
+ case nbsc: NBitsWidth_BitsCharset =>
+ trd.schemaDefinitionError("Only encodings with byte-sized code units are allowed to be specified using a runtime-valued expression. " +
+ "Encodings with 7 or fewer bits in their code units must be specified as a literal encoding name in the DFDL schema. " +
+ "The encoding found was '%s'.", bitsCharset.name)
+ case _ => // do nothing
+ }
+ }
+
override def compute(state: ParseOrUnparseState) = {
val encString = encodingEv.evaluate(state)
val cs = CharsetUtils.getCharset(encString)
Assert.invariant(cs ne null)
- new DFDLCharset(cs.name.toUpperCase)
+ val dcs = CharsetUtils.getCharset(cs.name.toUpperCase)
+ if (!encodingEv.isConstant) checkCharset(state, dcs)
+ dcs
}
}
@@ -128,20 +141,20 @@ class FillByteEv(fillByteRaw: String, charsetEv: CharsetEv, val trd: TermRuntime
val cookedFillByte = FillByteCooker.cook(fillByteRaw, trd, true)
Assert.invariant(cookedFillByte.length == 1)
- val dfdlCharset = charsetEv.evaluate(state)
- dfdlCharset.charset match {
- case _: NonByteSizeCharset => {
+ val bitsCharset = charsetEv.evaluate(state)
+ bitsCharset match {
+ case _: NBitsWidth_BitsCharset => {
state.SDE("The fillByte property cannot be specified as a" +
" character ('%s') when the dfdl:encoding property is '%s' because that" +
- " encoding is not a single-byte character set.", fillByteRaw, dfdlCharset.charsetName)
+ " encoding is not a single-byte character set.", fillByteRaw, bitsCharset.name)
}
- case cs => {
- val bytes = cookedFillByte.getBytes(cs)
+ case cs: BitsCharsetWrappingJavaCharset => {
+ val bytes = cookedFillByte.getBytes(cs.javaCharset)
Assert.invariant(bytes.length > 0)
if (bytes.length > 1) {
state.SDE("The fillByte property must be a single-byte" +
" character, but for encoding '%s' the specified character '%s'" +
- " occupies %d bytes", dfdlCharset.charsetName, cookedFillByte, bytes.length)
+ " occupies %d bytes", bitsCharset.name, cookedFillByte, bytes.length)
}
bytes(0).toInt
}
@@ -152,25 +165,3 @@ class FillByteEv(fillByteRaw: String, charsetEv: CharsetEv, val trd: TermRuntime
}
-class CheckEncodingEv(t: TermRuntimeData, alignmentInBits: Integer, charsetEv: CharsetEv)
- extends Evaluatable[Ok](t)
- with InfosetCachedEvaluatable[Ok] {
-
- override lazy val runtimeDependencies = List(charsetEv)
-
- final protected def compute(state: ParseOrUnparseState): Ok = {
-
- if (!charsetEv.isConstant) {
- val dfdlCS = charsetEv.evaluate(state)
- dfdlCS.charset match {
- case nbsc: NonByteSizeCharset =>
- t.schemaDefinitionError("Only encodings with byte-sized code units are allowed to be specified using a runtime-valued expression. " +
- "Encodings with 7 or fewer bits in their code units must be specified as a literal encoding name in the DFDL schema. " +
- "The encoding found was '%s'.", dfdlCS.charsetName)
- case _ => // do nothing
- }
- }
-
- Ok
- }
-}
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/Evaluatable.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/Evaluatable.scala
index 90e31d9..29ba17c 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/Evaluatable.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/Evaluatable.scala
@@ -76,129 +76,6 @@ object EvalCache {
}
}
-/**
- * Potentially runtime-evaluated things are instances of this base.
- */
-
-trait EvaluatableBase[+T <: AnyRef] extends Serializable {
-
- protected def rd: RuntimeData
-
- /**
- * Please override this.
- */
- def toBriefXML(depth: Int = -1) = toString
-
- /**
- * QName to be associated with this evaluatable.
- *
- * If the Evaluatable is for a DFDL property this should be dfdl:propertyName e.g., dfdl:terminator, with
- * the prefix DFDL being the prefix associated with the DFDL namespace.
- *
- * If the Evaluatable is for some other computation, e.g., for the CharsetEncoder. Then daf:encoder or other
- * useful identifier.
- *
- * This QName is used in the XML representation of these Evaluatables, if they are to be displayed, which is
- * mostly for various debug purposes.
- */
- lazy val qName: NamedQName = {
- val local = Misc.toInitialLowerCaseUnlessAllUpperCase(
- Misc.stripSuffix(Misc.getNameFromClass(this), "Ev"))
- GlobalQName(None, local, NoNamespace)
- }
-
- type State = ParseOrUnparseState
-
- /**
- * Important - Evalutables MUST declare all evaluatables they depend on. But only those
- * they directly depend on. (Not the dependents of the dependents...)
- */
- def runtimeDependencies: Seq[EvaluatableBase[AnyRef]]
-
- /**
- * The compute method is called at compilation time to determine if a static value can be returned.
- * If so the Ev will store and return that value every time evaluate is called. If not then this
- * will get called at runtime to compute the value.
- */
- protected def compute(state: State): T
-
- /**
- * The "apply" method is the "evaluate" routine. This and isConstant are the public operations on
- * this class.
- */
- def apply(state: State): T
-
- /**
- * The apply method is the "evaluate" function.
- */
- final def evaluate(state: State): T = apply(state)
-
- /**
- * If the object has been compiled, was it a constant or not?
- */
- def isConstant: Boolean
-
- def compile(): Option[T] = {
- val compState = new CompileState(rd, Nope)
- compile(compState)
- }
- /**
- * Been compiled yet?
- */
- var isCompiled = false
- final def ensureCompiled {
- if (!isCompiled)
- Assert.invariantFailed("not compiled Ev: " + this.qName)
- }
-
- /**
- * returns None if the value is not constant at compile time,
- * otherwise returns the constant value.
- */
- def compile(state: CompileState): Option[T] = {
- if (isCompiled) throw new IllegalStateException("already compiled")
-
- // must assign before call to apply since that requires it to be compiled
- // (the first evaluation is compilation)
- isCompiled = true
-
- //
- // detonation chamber. Evaluate it. Did it blow up because it needs
- // data, not just static information, to succeed?
- //
- val result = try {
- val v = apply(state)
- Some(v)
- } catch {
- //
- // Really what this should be catching are the special-purpose exceptions thrown
- // by the Infoset data-accessing API to indicate no data being found.
- //
- // However, for unparsing outputValueCalc this needs to distinguish whether we've determined
- // that the OVC expression is non-constant, from a failure at runtime.
- //
- // So non-constant is detected based on it failing by accessing either data or the infoset
- // when evaluated with a fake state that has neither. But SDE means there is something wrong
- // with the expression, so we can't absorb those.
- //
- // Thrown if we're trying to navigate from parent to child and the child doesn't exist.
- // or there is no data, etc.
- case e: ExpressionEvaluationException => None
- case e: InfosetException => None
- case e: VariableException => None
- }
- result
- }
-
- /**
- * Convenience method
- *
- * Use by way of
- * override lazy val qName = dafName("foobar")
- */
- protected def dafName(local: String) = GlobalQName(Some("dafint"), local, XMLUtils.dafintURI)
-}
-
trait InfosetCachedEvaluatable[T <: AnyRef] { self: Evaluatable[T] =>
protected def getCachedOrComputeAndCache(state: State): T = {
@@ -298,9 +175,76 @@ trait NoCacheEvaluatable[T <: AnyRef] { self: Evaluatable[T] =>
* Evaluatable - things that could be runtime-valued, but also could be compile-time constants
* are instances of Ev.
*/
-abstract class Evaluatable[+T <: AnyRef](protected val rd: RuntimeData, qNameArg: NamedQName = null) extends EvaluatableBase[T] {
+abstract class Evaluatable[+T <: AnyRef](protected val rd: RuntimeData, qNameArg: NamedQName = null)
+ extends Serializable {
+
+ type State = ParseOrUnparseState
+
+ /**
+ * Important - Evalutables MUST declare all evaluatables they depend on. But only those
+ * they directly depend on. (Not the dependents of the dependents...)
+ */
+ def runtimeDependencies: Seq[Evaluatable[AnyRef]]
/**
+ * Been compiled yet?
+ */
+ private var isCompiled_ = false
+
+ @inline final def isCompiled = isCompiled_
+
+ @inline final def ensureCompiled {
+ if (!isCompiled)
+ Assert.invariantFailed("not compiled Ev: " + this.qName)
+ }
+
+ /**
+ * returns None if the value is not constant at compile time,
+ * otherwise returns the constant value.
+ */
+ private def compileTimeEvaluate(state: CompileState): Maybe[T] = {
+ if (isCompiled) throw new IllegalStateException("already compiled")
+
+ // must assign before call to apply since that requires it to be compiled
+ // (the first evaluation is compilation)
+ isCompiled_ = true
+
+ //
+ // detonation chamber. Evaluate it. Did it blow up because it needs
+ // data, not just static information, to succeed?
+ //
+ val result = try {
+ val v = evaluate(state)
+ One(v)
+ } catch {
+ //
+ // Really what this should be catching are the special-purpose exceptions thrown
+ // by the Infoset data-accessing API to indicate no data being found.
+ //
+ // However, for unparsing outputValueCalc this needs to distinguish whether we've determined
+ // that the OVC expression is non-constant, from a failure at runtime.
+ //
+ // So non-constant is detected based on it failing by accessing either data or the infoset
+ // when evaluated with a fake state that has neither. But SDE means there is something wrong
+ // with the expression, so we can't absorb those.
+ //
+ // Thrown if we're trying to navigate from parent to child and the child doesn't exist.
+ // or there is no data, etc.
+ case e: ExpressionEvaluationException => Nope
+ case e: InfosetException => Nope
+ case e: VariableException => Nope
+ }
+ result
+ }
+
+ /**
+ * Convenience method
+ *
+ * Use by way of
+ * override lazy val qName = dafName("foobar")
+ */
+ protected def dafName(local: String) = GlobalQName(Some("dafint"), local, XMLUtils.dafintURI)
+ /**
* Override if this evaluatable needs to use a different evaluate mode
* for unparsing.
*
@@ -308,18 +252,30 @@ abstract class Evaluatable[+T <: AnyRef](protected val rd: RuntimeData, qNameArg
*/
protected def maybeUseUnparserMode: Maybe[EvalMode] = Maybe(UnparserNonBlocking)
- override def toString = "(%s@%x, %s)".format(qName, this.hashCode(), (if (constValue.isDefined) "constant: " + constValue.value else "runtime"))
+ override def toString = "(%s@%x, %s)".format(qName, this.hashCode(), (if (isConstant) "constant: " + constValue else "runtime"))
- override def toBriefXML(depth: Int = -1): String = if (constValue.isDefined) constValue.value.toString else super.toString
+ def toBriefXML(depth: Int = -1): String = if (isConstant) constValue.toString else toString
- override lazy val qName = if (qNameArg eq null) dafName(Misc.getNameFromClass(this)) else qNameArg
+ /**
+ * QName to be associated with this evaluatable.
+ *
+ * If the Evaluatable is for a DFDL property this should be dfdl:propertyName e.g., dfdl:terminator, with
+ * the prefix DFDL being the prefix associated with the DFDL namespace.
+ *
+ * If the Evaluatable is for some other computation, e.g., for the CharsetEncoder. Then daf:encoder or other
+ * useful identifier.
+ *
+ * This QName is used in the XML representation of these Evaluatables, if they are to be displayed, which is
+ * mostly for various debug purposes.
+ */
+ lazy val qName = if (qNameArg eq null) dafName(Misc.getNameFromClass(this)) else qNameArg
protected def getCachedOrComputeAndCache(state: State): T
- final override def apply(state: State): T = {
- ensureCompiled
+ protected def compute(state: State): T
- if (isConstant) constValue.get
+ final def evaluate(state: State): T = {
+ if (isConstant) constValue
else {
state match {
case _: CompileState => {
@@ -336,17 +292,21 @@ abstract class Evaluatable[+T <: AnyRef](protected val rd: RuntimeData, qNameArg
}
}
- private var constValue_ : Option[AnyRef] = None
- protected def constValue = constValue_.asInstanceOf[Option[T]]
+ private var constValue_ : Maybe[AnyRef] = Nope
- def optConstant = {
- ensureCompiled
- constValue
- }
+ /**
+ * Preferred for use in the runtime.
+ */
+ @inline final def maybeConstant = constValue_.asInstanceOf[Maybe[T]]
+ @inline final def isConstant = constValue_.isDefined
+ @inline final def constValue = maybeConstant.get
- def isConstant = {
- ensureCompiled
- constValue.isDefined
+ /**
+ * Schema compiler wants to use map call, so we need a scala option type
+ * for that. So this variant supplies that.
+ */
+ @inline final def optConstant = {
+ maybeConstant.toScalaOption
}
/**
@@ -354,12 +314,18 @@ abstract class Evaluatable[+T <: AnyRef](protected val rd: RuntimeData, qNameArg
* This is determined by actually evaluating it, passing a special CompileState
* that errors out when data access to runtime-valued data is attempted.
*/
- final override def compile(state: CompileState) = {
- val y = super.compile(state)
+ final def compile(state: CompileState): Maybe[T] = {
+ val y = compileTimeEvaluate(state)
+ // just by getting here - and not throwing, we know it's a constant.
constValue_ = y
y
}
+ final def compile(): Maybe[T] = {
+ val compState = new CompileState(rd, Nope)
+ compile(compState)
+ }
+
/**
* Creates an XML-like string representation.
*
@@ -400,7 +366,7 @@ abstract class Evaluatable[+T <: AnyRef](protected val rd: RuntimeData, qNameArg
* even considered as a possibility for a constant.
*/
-//abstract class Rv[+T <: AnyRef](rd: RuntimeData) extends EvaluatableBase[T](rd) {
+//abstract class Rv[+T <: AnyRef](rd: RuntimeData) extends Evaluatable[T](rd) {
//
// final override def isConstant = false // this is always to be computed
//
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/ProcessorBases.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/ProcessorBases.scala
index a9d393b..75fdbdc 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/ProcessorBases.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/ProcessorBases.scala
@@ -52,12 +52,15 @@ object Processor {
}
}
- private def ensureCompiled(ev: EvaluatableBase[AnyRef]) {
+ private def ensureCompiled(ev: Evaluatable[AnyRef]) {
ev.ensureCompiled
ev.runtimeDependencies.foreach { ensureCompiled }
}
}
+/**
+ * Captures common members for any processor, parser or unparser.
+ */
trait Processor
extends ToBriefXMLImpl
with Logging
@@ -68,10 +71,69 @@ trait Processor
def runtimeDependencies: Seq[Evaluatable[AnyRef]]
var isInitialized: Boolean = false
+
+ /**
+ * True if alignment, bit/byte order, and other aspects of real data on the data stream
+ * are relevant.
+ *
+ * True for primitive processors that actually touch the data stream, false
+ * for NoData, and for combinators.
+ *
+ * This enables an optimization in the runtime that doesn't evaluate
+ * expensive expressions for values or checking of encoding, byteOrder, bitOrder
+ * and so forth except for processors where it matters because they actually
+ * interact with the data stream.
+ */
+ def isPrimitive: Boolean
}
+/**
+ * A Prim or Primitive processor does not orchestrate the operation of
+ * other processors, it actually does the work of parsing/unparsing.
+ *
+ * Most PrimProcessor actually manipulate data to/from the data stream.
+ * Some (NoData) do not.
+ */
trait PrimProcessor extends Processor {
override def childProcessors: Seq[Processor] = Nil
+
+ /**
+ * True if alignment, bit/byte order, and other aspects of real data on the data stream
+ * are relevant.
+ *
+ * True for primitive processors that actually touch the data stream, false
+ * otherwise.
+ */
+ override def isPrimitive = true
+}
+
+/**
+ * A PrimProcessor which does other work than manipulating the data, such
+ * as evaluating DPath expressions.
+ */
+trait PrimProcessorNoData extends Processor {
+ override def childProcessors: Seq[Processor] = Nil
+
+ /**
+ * False because NoData processors don't touch the data stream.
+ */
+ override def isPrimitive = false
+}
+
+/**
+ * A combinator is a processor that orchestrates the operation of
+ * other processors.
+ *
+ * Combinators must be pure. They cannot both touch the data stream directly and
+ * also orchestrate other processors.
+ */
+trait CombinatorProcessor extends Processor {
+
+ /**
+ * False because combinators don't touch the data stream themselves. The
+ * processors they call do that work.
+ */
+ override final def isPrimitive = false
}
/** must mixin to all processors that deal with text */
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/ProcessorStateBases.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/ProcessorStateBases.scala
index 85e9fee..d464852 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/ProcessorStateBases.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/ProcessorStateBases.scala
@@ -68,8 +68,12 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.UTF16Width
import edu.illinois.ncsa.daffodil.processors.charset.CoderInfo
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.EncodingErrorPolicy
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.ByteOrder
-import java.nio.charset.CharsetDecoder
-import edu.illinois.ncsa.daffodil.io.NonByteSizeCharset
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharsetDecoder
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharsetEncoder
+import edu.illinois.ncsa.daffodil.processors.unparsers.UState
+import edu.illinois.ncsa.daffodil.processors.parsers.ParserBitOrderChecks
+import edu.illinois.ncsa.daffodil.processors.parsers.PState
+import edu.illinois.ncsa.daffodil.processors.unparsers.UnparserBitOrderChecks
/**
* Trait mixed into the PState.Mark object class and the ParseOrUnparseState
@@ -154,63 +158,110 @@ abstract class ParseOrUnparseState protected (
/*
* Implement the FormatInfo trait needed by the I/O layer.
*/
- final def replacingDecoder: CharsetDecoder = decoderEntry.replacingCoder
- final def reportingDecoder: CharsetDecoder = decoderEntry.reportingCoder
- final def binaryFloatRep: BinaryFloatRep = simpleElement.erd.maybeBinaryFloatRepEv.get.evaluate(this)
+
+ /*
+ * Slots that cache the value of these FormatInfo members so that
+ * they are recomputed only once per call to parse() or unparse() method.
+ */
+ private var binaryFloatRepCache: BinaryFloatRep = null
+ private var bitOrderCache: BitOrder = null
+ private var byteOrderCache: ByteOrder = null
+ private var maybeCachedFillByte: MaybeInt = MaybeInt.Nope
+ private var decoderCache: BitsCharsetDecoder = null
+ private var encoderCache: BitsCharsetEncoder = null
+ private var decoderCacheEntry_ : DecoderInfo = null
+ private var encoderCacheEntry_ : EncoderInfo = null
+
+ /**
+ * Must call after all processors so as to recompute the formatInfo slots
+ * for the next parser - since the values could change.
+ */
+ final def resetFormatInfoCaches(): Unit = {
+ binaryFloatRepCache = null
+ bitOrderCache = null
+ byteOrderCache = null
+ maybeCachedFillByte = MaybeInt.Nope
+ decoderCache = null
+ encoderCache = null
+ decoderCacheEntry_ = null
+ encoderCacheEntry_ = null
+ }
+
+ final def replacingDecoder: BitsCharsetDecoder = decoderEntry.replacingCoder
+ final def reportingDecoder: BitsCharsetDecoder = decoderEntry.reportingCoder
+
+ final def binaryFloatRep: BinaryFloatRep = {
+ if (binaryFloatRepCache eq null) {
+ binaryFloatRepCache = simpleElement.erd.maybeBinaryFloatRepEv.get.evaluate(this)
+ }
+ binaryFloatRepCache
+ }
private def runtimeData = processor.context
private def termRuntimeData = runtimeData.asInstanceOf[TermRuntimeData]
+ protected def checkBitOrder(): Unit
+
/**
* Returns bit order. If text, this is the bit order for the character set
* encoding. If binary, this is the bitOrder property value.
*/
final def bitOrder: BitOrder = {
- val res = processor match {
- case txtProc: TextProcessor =>
- encoder.charset() match {
- case nbs: NonByteSizeCharset => nbs.requiredBitOrder
- case _ => BitOrder.MostSignificantBitFirst
+ if (bitOrderCache eq null) {
+ val res = processor match {
+ case txtProc: TextProcessor =>
+ encoder.bitsCharset.requiredBitOrder
+ case _ => processor.context match {
+ case trd: TermRuntimeData => trd.defaultBitOrder
+ case ntrd: NonTermRuntimeData =>
+ Assert.usageError("Cannot ask for bitOrder for non-terms - NonTermRuntimeData: " + ntrd)
}
- case _ => processor.context match {
- case trd: TermRuntimeData => trd.defaultBitOrder
- case ntrd: NonTermRuntimeData =>
- Assert.usageError("Cannot ask for bitOrder for non-terms - NonTermRuntimeData: " + ntrd)
}
+ bitOrderCache = res
+ checkBitOrder()
}
- res
+ bitOrderCache
}
final def byteOrder: ByteOrder = {
- runtimeData match {
- case erd: ElementRuntimeData => erd.maybeByteOrderEv.get.evaluate(this)
- case mgrd: ModelGroupRuntimeData => {
- //
- // Model Groups can't have byte order.
- // However, I/O layer still requests it because alignment regions
- // use skip, which ultimately uses getLong/putLong, which asks for
- // byteOrder.
- //
- // A model group DOES care about bit order for its alignment regions,
- // and for the charset encoding of say, initiators or prefix separators.
- // A bitOrder change requires that we check the new bitOrder against the
- // byte order to insure compatibility. (byteOrder can be an expression),
- // so of necessity, we also need byte order. However, if byte order isn't defined
- // we can assume littleEndian since that works with all bit orders.
- // (Big endian only allows MSBF bit order)
- //
- ByteOrder.LittleEndian
+ if (byteOrderCache eq null) {
+ val bo = runtimeData match {
+ case erd: ElementRuntimeData => erd.maybeByteOrderEv.get.evaluate(this)
+ case mgrd: ModelGroupRuntimeData => {
+ //
+ // Model Groups can't have byte order.
+ // However, I/O layer still requests it because alignment regions
+ // use skip, which ultimately uses getLong/putLong, which asks for
+ // byteOrder.
+ //
+ // A model group DOES care about bit order for its alignment regions,
+ // and for the charset encoding of say, initiators or prefix separators.
+ // A bitOrder change requires that we check the new bitOrder against the
+ // byte order to insure compatibility. (byteOrder can be an expression),
+ // so of necessity, we also need byte order. However, if byte order isn't defined
+ // we can assume littleEndian since that works with all bit orders.
+ // (Big endian only allows MSBF bit order)
+ //
+ ByteOrder.LittleEndian
+ }
+ case _ => Assert.usageError("byte order of non term: " + runtimeData)
}
- case _ => Assert.usageError("byte order of non term: " + runtimeData)
+ byteOrderCache = bo
}
+ byteOrderCache
}
- final def maybeCharWidthInBits: MaybeInt = { coderCacheEntry_.maybeCharWidthInBits }
- final def encodingMandatoryAlignmentInBits: Int = { decoder; coderCacheEntry_.encodingMandatoryAlignmentInBits }
+ final def maybeCharWidthInBits: MaybeInt = { coderEntry.maybeCharWidthInBits }
+ final def encodingMandatoryAlignmentInBits: Int = { coderEntry.encodingMandatoryAlignmentInBits }
final def maybeUTF16Width: Maybe[UTF16Width] = termRuntimeData.encodingInfo.maybeUTF16Width
- final def fillByte: Byte = termRuntimeData.maybeFillByteEv.get.evaluate(this).toByte
- final def decoder = {
+ final def fillByte: Byte = {
+ if (maybeCachedFillByte.isEmpty)
+ maybeCachedFillByte = MaybeInt(termRuntimeData.maybeFillByteEv.get.evaluate(this).toInt)
+ maybeCachedFillByte.get.toByte
+ }
+
+ private def getDecoder() = {
val de = decoderEntry
if (encodingErrorPolicy eq EncodingErrorPolicy.Error)
de.reportingCoder
@@ -218,7 +269,13 @@ abstract class ParseOrUnparseState protected (
de.replacingCoder
}
- final def encoder = {
+ final def decoder = {
+ if (decoderCache eq null)
+ decoderCache = getDecoder()
+ decoderCache
+ }
+
+ private def getEncoder() = {
val ee = encoderEntry
if (encodingErrorPolicy eq EncodingErrorPolicy.Error)
ee.reportingCoder
@@ -226,29 +283,46 @@ abstract class ParseOrUnparseState protected (
ee.replacingCoder
}
+ final def encoder = {
+ if (encoderCache eq null)
+ encoderCache = getEncoder()
+ encoderCache
+ }
+
final def encodingErrorPolicy: EncodingErrorPolicy = {
val eep = termRuntimeData.encodingInfo.defaultEncodingErrorPolicy
eep
}
+ private def coderEntry: CoderInfo = {
+ if (this.isInstanceOf[UState]) encoderEntry
+ else decoderEntry
+ }
+
private def decoderEntry = {
- val nextEntry = termRuntimeData.encodingInfo.getDecoderInfo(this)
- if (coderCacheEntry_ == null || coderCacheEntry_ != nextEntry) {
- coderCacheEntry_ = nextEntry
+ if (decoderCacheEntry_ eq null) {
+ val nextEntry = termRuntimeData.encodingInfo.getDecoderInfo(this)
+ decoderCacheEntry_ = nextEntry
+ if (this.processor.isPrimitive)
+ if (decoderCacheEntry_.encodingMandatoryAlignmentInBitsArg != 1)
+ if (this.bitPos1b % 8 != 1)
+ ParserBitOrderChecks.checkParseBitOrder(this.asInstanceOf[PState])
}
- coderCacheEntry_.asInstanceOf[DecoderInfo]
+ decoderCacheEntry_
}
private def encoderEntry = {
- val nextEntry = termRuntimeData.encodingInfo.getEncoderInfo(this)
- if (coderCacheEntry_ == null || coderCacheEntry_ != nextEntry) {
- coderCacheEntry_ = nextEntry
+ if (encoderCacheEntry_ eq null) {
+ val nextEntry = termRuntimeData.encodingInfo.getEncoderInfo(this)
+ encoderCacheEntry_ = nextEntry
+ if (this.processor.isPrimitive)
+ if (encoderCacheEntry_.encodingMandatoryAlignmentInBitsArg != 1)
+ if (this.bitPos1b % 8 != 1)
+ UnparserBitOrderChecks.checkUnparseBitOrder(this.asInstanceOf[UState])
}
- coderCacheEntry_.asInstanceOf[EncoderInfo]
+ encoderCacheEntry_
}
- private var coderCacheEntry_ : CoderInfo = _
-
/**
* Variable map provides access to variable bindings.
*/
@@ -295,14 +369,14 @@ abstract class ParseOrUnparseState protected (
def setSuccess() {
_processorStatus = Success
}
-
+
/**
* Used when errors are caught by interactive debugger expression evaluation.
* We don't want to accumulate the diagnostics that we're suppressing.
*/
final def suppressDiagnosticAndSucceed(d: Diagnostic) {
Assert.usage(diagnostics.contains(d))
- diagnostics = diagnostics.filterNot{ _ eq d}
+ diagnostics = diagnostics.filterNot { _ eq d }
setSuccess()
}
@@ -457,4 +531,6 @@ final class CompileState(trd: RuntimeData, maybeDataProc: Maybe[DataProcessor])
// Members declared in edu.illinois.ncsa.daffodil.processors.StateForDebugger
def currentLocation: DataLocation = Assert.usageError("Not to be used.")
+
+ protected def checkBitOrder(): Unit = {}
}
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/SchemaSetRuntimeData.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/SchemaSetRuntimeData.scala
index d2ca5bd..91722ce 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/SchemaSetRuntimeData.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/SchemaSetRuntimeData.scala
@@ -52,4 +52,4 @@ final class SchemaSetRuntimeData(
override def schemaFileLocation = elementRuntimeData.schemaFileLocation
override def SDE(str: String, args: Any*) = elementRuntimeData.SDE(str, args)
-}
\ No newline at end of file
+}
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/SuspendableOperation.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/SuspendableOperation.scala
index 32cc35a..e57c2ad 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/SuspendableOperation.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/SuspendableOperation.scala
@@ -40,7 +40,6 @@ import edu.illinois.ncsa.daffodil.api.Diagnostic
import edu.illinois.ncsa.daffodil.util.Maybe
import edu.illinois.ncsa.daffodil.util.Maybe._
import edu.illinois.ncsa.daffodil.infoset.RetryableException
-import edu.illinois.ncsa.daffodil.processors.unparsers.Unparser
/**
* SuspendableOperation is used for suspending and retrying things that aren't
@@ -106,16 +105,6 @@ trait SuspendableOperation
}
}
-trait SuspendableUnparser
- extends Unparser {
-
- protected def suspendableOperation: SuspendableOperation
-
- final def unparse(state: UState): Unit = {
- suspendableOperation.run(state)
- }
-}
-
class SuspendableOperationException(m: String)
extends Diagnostic(Nope, Nope, Nope, Maybe(m)) {
override def isError = true
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/Parser.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/Parser.scala
index 5799447..b02a074 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/Parser.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/Parser.scala
@@ -39,7 +39,7 @@ import edu.illinois.ncsa.daffodil.processors.RuntimeData
/**
* Parent class of all DFA text parsers.
*/
-abstract class Parser extends Serializable {
+abstract class DFAParser extends Serializable {
def name: String
def info: String
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/TextDelimitedParser.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/TextDelimitedParser.scala
index 931b0f3..1b78e07 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/TextDelimitedParser.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/TextDelimitedParser.scala
@@ -49,7 +49,7 @@ abstract class TextDelimitedParserBase(
override val justificationTrim: TextJustificationType.Type,
override val parsingPadChar: MaybeChar,
override val context: TermRuntimeData)
- extends Parser with PaddingRuntimeMixin {
+ extends DFAParser with PaddingRuntimeMixin {
private lazy val padCharInfo = if (parsingPadChar.isDefined) parsingPadChar.toString else "NONE"
lazy val info: String = "justification='" + justificationTrim + "', padChar='" + padCharInfo + "'"
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/TextPaddingParser.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/TextPaddingParser.scala
index 224b1c5..4f2a762 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/TextPaddingParser.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/TextPaddingParser.scala
@@ -42,10 +42,10 @@ import edu.illinois.ncsa.daffodil.io.FormatInfo
class TextPaddingParser(val padChar: Char,
override val context: TermRuntimeData)
- extends Parser {
+ extends DFAParser {
- lazy val name: String = "TextPaddingParser"
- lazy val info: String = "padChar='" + padChar + "'"
+ override lazy val name: String = "TextPaddingParser"
+ override lazy val info: String = "padChar='" + padChar + "'"
val paddingDFA = CreatePaddingDFA(padChar, context)
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/TextParser.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/TextParser.scala
index 413350a..7608a6f 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/TextParser.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/dfa/TextParser.scala
@@ -41,7 +41,7 @@ import edu.illinois.ncsa.daffodil.io.FormatInfo
class TextParser(
override val context: TermRuntimeData)
- extends Parser {
+ extends DFAParser {
override lazy val name: String = "TextParser"
override lazy val info: String = "" // Nothing additional to add here
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/AssertPatternParsers.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/AssertPatternParsers.scala
index 0318d70..d75a21f 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/AssertPatternParsers.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/AssertPatternParsers.scala
@@ -40,16 +40,17 @@ import edu.illinois.ncsa.daffodil.util.OnStack
abstract class AssertPatternParserBase(
eName: String,
kindString: String,
- rd: TermRuntimeData,
+ override val context: TermRuntimeData,
testPattern: String,
message: String)
- extends PrimParserObject(rd) {
+ extends PrimParser {
+ override lazy val runtimeDependencies: Seq[Evaluatable[AnyRef]] = Nil
override def toBriefXML(depthLimit: Int = -1) = {
"<" + kindString + ">" + testPattern + "</" + kindString + ">"
}
- // private lazy val compiledPattern = ScalaPatternParser.compilePattern(testPattern, rd)
+ // private lazy val compiledPattern = ScalaPatternParser.compilePattern(testPattern, context)
lazy val pattern = ("(?s)" + testPattern).r.pattern // imagine a really big expensive pattern to compile.
object withMatcher extends OnStack[Matcher](pattern.matcher(""))
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/BinaryNumberParsers.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/BinaryNumberParsers.scala
index b8f1e12..575e9f4 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/BinaryNumberParsers.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/BinaryNumberParsers.scala
@@ -39,8 +39,9 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.YesNo
import java.lang.{ Long => JLong, Number => JNumber, Double => JDouble, Float => JFloat }
import edu.illinois.ncsa.daffodil.processors.ParseOrUnparseState
-class BinaryFloatParser(e: ElementRuntimeData)
- extends PrimParserObject(e) {
+class BinaryFloatParser(override val context: ElementRuntimeData)
+ extends PrimParser {
+ override lazy val runtimeDependencies = Nil
def parse(start: PState): Unit = {
val dis = start.dataInputStream
@@ -55,8 +56,9 @@ class BinaryFloatParser(e: ElementRuntimeData)
}
}
-class BinaryDoubleParser(e: ElementRuntimeData)
- extends PrimParserObject(e) {
+class BinaryDoubleParser(override val context: ElementRuntimeData)
+ extends PrimParser {
+ override lazy val runtimeDependencies = Nil
def parse(start: PState): Unit = {
val dis = start.dataInputStream
@@ -81,8 +83,9 @@ class BinaryDecimalRuntimeLengthParser(val e: ElementRuntimeData, signed: YesNo,
with HasRuntimeExplicitLength {
}
-abstract class BinaryDecimalParserBase(e: ElementRuntimeData, signed: YesNo, binaryDecimalVirtualPoint: Int)
- extends PrimParserObject(e) {
+abstract class BinaryDecimalParserBase(override val context: ElementRuntimeData, signed: YesNo, binaryDecimalVirtualPoint: Int)
+ extends PrimParser {
+ override lazy val runtimeDependencies = Nil
protected def getBitLength(s: ParseOrUnparseState): Int
@@ -113,8 +116,9 @@ class BinaryIntegerKnownLengthParser(e: ElementRuntimeData, signed: Boolean, val
with HasKnownLengthInBits {
}
-abstract class BinaryIntegerBaseParser(e: ElementRuntimeData, signed: Boolean)
- extends PrimParserObject(e) {
+abstract class BinaryIntegerBaseParser(override val context: ElementRuntimeData, signed: Boolean)
+ extends PrimParser {
+ override lazy val runtimeDependencies = Nil
protected def getBitLength(s: ParseOrUnparseState): Int
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/DelimitedParsers.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/DelimitedParsers.scala
index 3de07f8..74da01f 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/DelimitedParsers.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/DelimitedParsers.scala
@@ -49,24 +49,25 @@ import edu.illinois.ncsa.daffodil.util.MaybeChar
import edu.illinois.ncsa.daffodil.util.DecimalUtils
import edu.illinois.ncsa.daffodil.processors.AllTerminatingMarkupDelimiterIterator
import passera.unsigned.ULong
+import edu.illinois.ncsa.daffodil.processors.charset.StandardBitsCharsets
class StringDelimitedParser(
- erd: ElementRuntimeData,
+ override val context: ElementRuntimeData,
justificationTrim: TextJustificationType.Type,
pad: MaybeChar,
textParser: TextDelimitedParserBase,
fieldDFAEv: FieldDFAParseEv,
isDelimRequired: Boolean)
- extends TextPrimParserObject(erd)
+ extends TextPrimParser
with CaptureParsingValueLength {
- override val runtimeDependencies = List(fieldDFAEv, erd.encInfo.charsetEv)
+ override val runtimeDependencies = List(fieldDFAEv, context.encInfo.charsetEv)
- override val charsetEv = erd.encInfo.charsetEv
+ override val charsetEv = context.encInfo.charsetEv
def processResult(parseResult: Maybe[dfa.ParseResult], state: PState): Unit = {
- if (!parseResult.isDefined) this.PE(state, "%s - %s - Parse failed.", this.toString(), erd.diagnosticDebugName)
+ if (!parseResult.isDefined) this.PE(state, "%s - %s - Parse failed.", this.toString(), context.diagnosticDebugName)
else {
val result = parseResult.get
val field = if (result.field.isDefined) result.field.get else ""
@@ -172,7 +173,7 @@ class HexBinaryDelimitedParser(
*/
override def processResult(parseResult: Maybe[dfa.ParseResult], state: PState): Unit = {
- Assert.invariant(erd.encodingInfo.isKnownEncoding && erd.encodingInfo.knownEncodingCharset.charset =:= StandardCharsets.ISO_8859_1)
+ Assert.invariant(erd.encodingInfo.isKnownEncoding && erd.encodingInfo.knownEncodingCharset =:= StandardBitsCharsets.ISO_8859_1)
if (!parseResult.isDefined) this.PE(state, "%s - %s - Parse failed.", this.toString(), erd.diagnosticDebugName)
else {
@@ -187,7 +188,6 @@ class HexBinaryDelimitedParser(
}
}
-
class PackedIntegerDelimitedParser(
erd: ElementRuntimeData,
textParser: TextDelimitedParserBase,
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/ElementCombinator1.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/ElementCombinator1.scala
index 767cfb7..17c732f 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/ElementCombinator1.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/ElementCombinator1.scala
@@ -56,7 +56,9 @@ abstract class ElementParserBase(
eBeforeParser: Maybe[Parser],
eParser: Maybe[Parser],
eAfterParser: Maybe[Parser])
- extends ParserObject(rd) {
+ extends CombinatorParser(rd) {
+
+ override lazy val runtimeDependencies = Nil
def move(pstate: PState): Unit // implement for different kinds of "moving over to next thing"
def parseBegin(pstate: PState): Unit
@@ -136,7 +138,6 @@ abstract class ElementParserBase(
}
parseBegin(pstate)
- ParserBitOrderChecks.checkParseBitOrder(pstate)
try {
// TODO: Performance/Maintainability - get rid of use of return statements.
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/ElementKindParsers.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/ElementKindParsers.scala
index 133652f..ecac818 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/ElementKindParsers.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/ElementKindParsers.scala
@@ -43,9 +43,11 @@ import edu.illinois.ncsa.daffodil.util.LogLevel
import edu.illinois.ncsa.daffodil.processors.TermRuntimeData
class ComplexTypeParser(rd: RuntimeData, bodyParser: Parser)
- extends ParserObject(rd) {
+ extends CombinatorParser(rd) {
override def nom = "ComplexType"
+ override lazy val runtimeDependencies = Nil
+
override lazy val childProcessors = Seq(bodyParser)
def parse(start: PState): Unit = {
@@ -64,8 +66,8 @@ class ComplexTypeParser(rd: RuntimeData, bodyParser: Parser)
* the internal/body parser has completed.
*/
class DelimiterStackParser(delimiters: Array[DelimiterParseEv],
- override val context: RuntimeData, bodyParser: Parser)
- extends Parser {
+ ctxt: RuntimeData, bodyParser: Parser)
+ extends CombinatorParser(ctxt) {
override lazy val childProcessors = List(bodyParser)
@@ -110,8 +112,8 @@ class DelimiterStackParser(delimiters: Array[DelimiterParseEv],
* cached, so upon exiting scope the cache must be invalidated.
*/
class DynamicEscapeSchemeParser(escapeScheme: EscapeSchemeParseEv,
- override val context: TermRuntimeData, bodyParser: Parser)
- extends Parser {
+ ctxt: TermRuntimeData, bodyParser: Parser)
+ extends CombinatorParser(ctxt) {
override lazy val childProcessors = Seq(bodyParser)
@@ -133,9 +135,11 @@ class DynamicEscapeSchemeParser(escapeScheme: EscapeSchemeParseEv,
}
class SequenceCombinatorParser(rd: TermRuntimeData, bodyParser: Parser)
- extends ParserObject(rd) {
+ extends CombinatorParser(rd) {
override def nom = "Sequence"
+ override lazy val runtimeDependencies = Nil
+
override lazy val childProcessors = Seq(bodyParser)
def parse(start: PState): Unit = {
@@ -155,9 +159,11 @@ class SequenceCombinatorParser(rd: TermRuntimeData, bodyParser: Parser)
* which has a more complicated unparser that differs from an AltCompUnparser.
*/
class ChoiceCombinatorParser(rd: TermRuntimeData, bodyParser: Parser)
- extends ParserObject(rd) {
+ extends CombinatorParser(rd) {
override def nom = "Choice"
+ override lazy val runtimeDependencies = Nil
+
override lazy val childProcessors = Seq(bodyParser)
def parse(start: PState): Unit = {
@@ -166,9 +172,11 @@ class ChoiceCombinatorParser(rd: TermRuntimeData, bodyParser: Parser)
}
class ChoiceDispatchCombinatorParser(rd: TermRuntimeData, dispatchKeyEv: ChoiceDispatchKeyEv, dispatchBranchKeyMap: Map[String, Parser])
- extends ParserObject(rd) {
+ extends CombinatorParser(rd) {
override def nom = "ChoiceDispatch"
+ override lazy val runtimeDependencies = Nil
+
override lazy val childProcessors = dispatchBranchKeyMap.values.toSeq
def parse(pstate: PState): Unit = {
@@ -203,10 +211,13 @@ class ChoiceDispatchCombinatorParser(rd: TermRuntimeData, dispatchKeyEv: ChoiceD
}
}
-class ArrayCombinatorParser(erd: ElementRuntimeData, bodyParser: Parser) extends ParserObject(erd) {
+class ArrayCombinatorParser(erd: ElementRuntimeData, bodyParser: Parser)
+ extends CombinatorParser(erd) {
override def nom = "Array"
override lazy val childProcessors = Seq(bodyParser)
+ override lazy val runtimeDependencies = Nil
+
def parse(start: PState): Unit = {
start.mpstate.arrayIndexStack.push(1L) // one-based indexing
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/ExpressionEvaluatingParsers.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/ExpressionEvaluatingParsers.scala
index 4ddbc26..a95da9f 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/ExpressionEvaluatingParsers.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/ExpressionEvaluatingParsers.scala
@@ -51,8 +51,10 @@ import edu.illinois.ncsa.daffodil.util.LogLevel
*/
abstract class ExpressionEvaluationParser(
expr: CompiledExpression[AnyRef],
- rd: RuntimeData)
- extends ParserObject(rd) {
+ override val context: RuntimeData)
+ extends PrimParserNoData {
+
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Nil
@@ -95,20 +97,24 @@ class SetVariableParser(expr: CompiledExpression[AnyRef], decl: VariableRuntimeD
}
class NewVariableInstanceStartParser(
- decl: RuntimeData)
- extends PrimParserObject(decl) {
- decl.notYetImplemented("newVariableInstance")
+ override val context: RuntimeData)
+ extends PrimParser {
+ context.notYetImplemented("newVariableInstance")
+ override lazy val runtimeDependencies = Nil
+
def parse(pstate: PState) = {
- decl.notYetImplemented("newVariableInstance")
+ context.notYetImplemented("newVariableInstance")
}
}
class NewVariableInstanceEndParser(
- decl: RuntimeData)
- extends PrimParserObject(decl) {
- decl.notYetImplemented("newVariableInstance")
+ override val context: RuntimeData)
+ extends PrimParser {
+ context.notYetImplemented("newVariableInstance")
+ override lazy val runtimeDependencies = Nil
+
def parse(pstate: PState) = {
- decl.notYetImplemented("newVariableInstance")
+ context.notYetImplemented("newVariableInstance")
}
}
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/NilParsers.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/NilParsers.scala
index 8767ad0..1fda9e8 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/NilParsers.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/NilParsers.scala
@@ -38,13 +38,14 @@ import edu.illinois.ncsa.daffodil.util.MaybeChar
abstract class LiteralNilOfSpecifiedLengthParserBase(
erd: ElementRuntimeData)
- extends TextPrimParserObject(erd)
+ extends TextPrimParser
with StringOfSpecifiedLengthMixin
with NilMatcherMixin {
private val eName = erd.name
override val runtimeDependencies = List(erd.encInfo.charsetEv)
+ override val context = erd
override val charsetEv = erd.encInfo.charsetEv
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/OptionalInfixSepParser.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/OptionalInfixSepParser.scala
index 87ce77f..431c9ad 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/OptionalInfixSepParser.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/OptionalInfixSepParser.scala
@@ -34,8 +34,8 @@ package edu.illinois.ncsa.daffodil.processors.parsers
import edu.illinois.ncsa.daffodil.processors.TermRuntimeData
-class OptionalInfixSepParser(override val context: TermRuntimeData, sepParser: Parser)
- extends Parser {
+class OptionalInfixSepParser(ctxt: TermRuntimeData, sepParser: Parser)
+ extends CombinatorParser(ctxt) {
override lazy val nom = "OptionalInfixSep"
override lazy val childProcessors = Seq(sepParser)
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PState.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PState.scala
index d6de3dd..9c625bf 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PState.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PState.scala
@@ -283,6 +283,10 @@ final class PState private (
// threadCheck()
dataInputStream.setDebugging(flag)
}
+
+ final override protected def checkBitOrder(): Unit = {
+ ParserBitOrderChecks.checkParseBitOrder(this)
+ }
}
object PState {
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PackedBinaryTraits.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PackedBinaryTraits.scala
index 41553e2..7bb7a7e 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PackedBinaryTraits.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PackedBinaryTraits.scala
@@ -32,18 +32,21 @@
package edu.illinois.ncsa.daffodil.processors.parsers
+import java.math.{ BigDecimal => JBigDecimal }
+import java.math.{ BigInteger => JBigInteger }
+import java.nio.charset.StandardCharsets
+
+import edu.illinois.ncsa.daffodil.equality.TypeEqual
+import edu.illinois.ncsa.daffodil.exceptions.Assert
import edu.illinois.ncsa.daffodil.processors.ElementRuntimeData
-import edu.illinois.ncsa.daffodil.processors.ParseOrUnparseState
import edu.illinois.ncsa.daffodil.processors.FieldDFAParseEv
+import edu.illinois.ncsa.daffodil.processors.ParseOrUnparseState
import edu.illinois.ncsa.daffodil.processors.TextJustificationType
+import edu.illinois.ncsa.daffodil.processors.charset.StandardBitsCharsets
+import edu.illinois.ncsa.daffodil.processors.dfa
import edu.illinois.ncsa.daffodil.processors.dfa.TextDelimitedParserBase
import edu.illinois.ncsa.daffodil.util.Maybe
-import edu.illinois.ncsa.daffodil.exceptions.Assert
-import edu.illinois.ncsa.daffodil.equality._
-import edu.illinois.ncsa.daffodil.processors.dfa
import edu.illinois.ncsa.daffodil.util.MaybeChar
-import java.math.{ BigInteger => JBigInteger, BigDecimal => JBigDecimal }
-import java.nio.charset.StandardCharsets
import passera.unsigned.ULong
trait PackedBinaryConversion {
@@ -52,10 +55,11 @@ trait PackedBinaryConversion {
}
abstract class PackedBinaryDecimalBaseParser(
- e: ElementRuntimeData,
+ override val context: ElementRuntimeData,
binaryDecimalVirtualPoint: Int)
- extends PrimParserObject(e)
+ extends PrimParser
with PackedBinaryConversion {
+ override lazy val runtimeDependencies = Nil
protected def getBitLength(s: ParseOrUnparseState): Int
@@ -79,10 +83,11 @@ abstract class PackedBinaryDecimalBaseParser(
}
abstract class PackedBinaryIntegerBaseParser(
- e: ElementRuntimeData,
+ override val context: ElementRuntimeData,
signed: Boolean = false)
- extends PrimParserObject(e)
+ extends PrimParser
with PackedBinaryConversion {
+ override lazy val runtimeDependencies = Nil
protected def getBitLength(s: ParseOrUnparseState): Int
@@ -117,7 +122,7 @@ abstract class PackedBinaryIntegerDelimitedBaseParser(
with PackedBinaryConversion {
override def processResult(parseResult: Maybe[dfa.ParseResult], state: PState): Unit = {
- Assert.invariant(e.encodingInfo.isKnownEncoding && e.encodingInfo.knownEncodingCharset.charset =:= StandardCharsets.ISO_8859_1)
+ Assert.invariant(e.encodingInfo.isKnownEncoding && e.encodingInfo.knownEncodingCharset =:= StandardBitsCharsets.ISO_8859_1)
if (!parseResult.isDefined) this.PE(state, "%s - %s - Parse failed.", this.toString(), e.diagnosticDebugName)
else {
@@ -143,7 +148,6 @@ abstract class PackedBinaryIntegerDelimitedBaseParser(
}
}
-
abstract class PackedBinaryDecimalDelimitedBaseParser(
e: ElementRuntimeData,
textParser: TextDelimitedParserBase,
@@ -166,7 +170,7 @@ abstract class PackedBinaryDecimalDelimitedBaseParser(
*/
override def processResult(parseResult: Maybe[dfa.ParseResult], state: PState): Unit = {
- Assert.invariant(e.encodingInfo.isKnownEncoding && e.encodingInfo.knownEncodingCharset.charset =:= StandardCharsets.ISO_8859_1)
+ Assert.invariant(e.encodingInfo.isKnownEncoding && e.encodingInfo.knownEncodingCharset =:= StandardBitsCharsets.ISO_8859_1)
if (!parseResult.isDefined) this.PE(state, "%s - %s - Parse failed.", this.toString(), e.diagnosticDebugName)
else {
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/Parser.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/Parser.scala
index 1bec6d0..9e53880 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/Parser.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/Parser.scala
@@ -48,17 +48,19 @@ import edu.illinois.ncsa.daffodil.util.LogLevel
import edu.illinois.ncsa.daffodil.util.Maybe.One
import edu.illinois.ncsa.daffodil.util.Misc
import edu.illinois.ncsa.daffodil.processors.TermRuntimeData
-import edu.illinois.ncsa.daffodil.processors.ModelGroupRuntimeData
+import edu.illinois.ncsa.daffodil.processors.Evaluatable
+import edu.illinois.ncsa.daffodil.processors.CombinatorProcessor
+import edu.illinois.ncsa.daffodil.processors.PrimProcessor
+import edu.illinois.ncsa.daffodil.processors.TextProcessor
+import edu.illinois.ncsa.daffodil.processors.PrimProcessorNoData
/**
* Encapsulates lower-level parsing with a uniform interface
*
- * A parser can have sub-parsers. See also PrimParser which is a parser with no sub-parsers.
- *
- * This trait is preferred (and PrimParser) over the ParserObject because this requires one to
- * explicitly manage runtimeDependencies, whereas ParserObject hides that detail, which isn't helpful.
+ * A parser can have sub-parsers. See also PrimParser which is a parser with no sub-parsers,
+ * and CombinatorParser, which is a parser with sub parsers.
*/
-trait Parser
+sealed trait Parser
extends Processor {
protected def parserName = Misc.getNameFromClass(this)
@@ -78,11 +80,6 @@ trait Parser
pstate.setProcessor(this)
if (pstate.dataProc.isDefined) pstate.dataProc.get.before(pstate, this)
try {
- pstate.processor.context match {
- case mgrd: ModelGroupRuntimeData =>
- ParserBitOrderChecks.checkParseBitOrder(pstate)
- case _ => // ok. Elements are checked elsewhere.
- }
parse(pstate)
} catch {
/*
@@ -110,22 +107,64 @@ trait Parser
* ProcessingError it throws a new SDE.
* */
case pe: ParseError => pstate.setFailed(pe)
+ } finally {
+ pstate.resetFormatInfoCaches()
}
if (pstate.dataProc.isDefined) pstate.dataProc.get.after(pstate, this)
pstate.setMaybeProcessor(savedParser)
}
}
-// Deprecated and to be phased out. Use the trait Parser instead.
-abstract class ParserObject(override val context: RuntimeData)
- extends Parser {
+/**
+ * A PrimParser is a parser that contains no child parsers.
+ * Combinators are NOT PrimParser
+ */
+trait PrimParser
+ extends PrimProcessor
+ with Parser
- override lazy val runtimeDependencies = Nil
+/**
+ * A parser which is "primitive" no sub-parsers, but which doesn't
+ * parses any data. Evaluates expressions, binds variables, evaluates
+ * asserts, etc.
+ */
+trait PrimParserNoData
+ extends PrimProcessorNoData
+ with Parser
+
+/**
+ * Mixed in as a marker for parsers that only operate on text.
+ */
+trait TextPrimParser
+ extends PrimParser
+ with TextProcessor
+
+/**
+ * Parser which does "Nada" (Nothing in Spanish)
+ *
+ * Used for optitonality in some cases, but is usually recognized and
+ * optimized out.
+ */
+final class NadaParser(override val context: RuntimeData)
+ extends PrimParserNoData {
+ override def runtimeDependencies: Seq[Evaluatable[AnyRef]] = Nil
+
+ override def toString = "Nada"
+
+ override def parse(start: PState): Unit = {
+ // do nothing
+ }
}
-class SeqCompParser(context: RuntimeData, val childParsers: Array[Parser])
- extends ParserObject(context) {
+abstract class CombinatorParser(override val context: RuntimeData)
+ extends Parser with CombinatorProcessor {
+ // override lazy val runtimeDependencies = Nil
+}
+
+final class SeqCompParser(context: RuntimeData, val childParsers: Array[Parser])
+ extends CombinatorParser(context) {
+ override lazy val runtimeDependencies = Nil
override def childProcessors = childParsers
override def nom = "seq"
@@ -146,8 +185,8 @@ class SeqCompParser(context: RuntimeData, val childParsers: Array[Parser])
}
class AltCompParser(ctxt: RuntimeData, val childParsers: Seq[Parser])
- extends ParserObject(ctxt) {
-
+ extends CombinatorParser(ctxt) {
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = childParsers
override def nom = "alt"
@@ -259,15 +298,18 @@ class AltCompParser(ctxt: RuntimeData, val childParsers: Seq[Parser])
}
-case class DummyParser(rd: TermRuntimeData) extends ParserObject(null) {
- def parse(pstate: PState): Unit = pstate.SDE("Parser for " + rd + " is not yet implemented.")
+case class DummyParser(override val context: TermRuntimeData)
+ extends PrimParserNoData {
+ override def runtimeDependencies: Seq[Evaluatable[AnyRef]] = Nil
+
+ def parse(pstate: PState): Unit = pstate.SDE("Parser for " + context + " is not yet implemented.")
override def childProcessors = Nil
override def toBriefXML(depthLimit: Int = -1) = "<dummy/>"
- override def toString = if (rd == null) "Dummy[null]" else "Dummy[" + rd + "]"
+ override def toString = if (context == null) "Dummy[null]" else "Dummy[" + context + "]"
}
-case class NotParsableParser(context: ElementRuntimeData) extends Parser {
+case class NotParsableParser(context: ElementRuntimeData) extends PrimParserNoData {
def parse(state: PState): Unit = {
// We can't use state.SDE because that needs the infoset to determine the
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PrimParser.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PrimParser.scala
deleted file mode 100644
index ee4429f..0000000
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PrimParser.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-/* Copyright (c) 2012-2014 Tresys Technology, LLC. All rights reserved.
- *
- * Developed by: Tresys Technology, LLC
- * http://www.tresys.com
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy of
- * this software and associated documentation files (the "Software"), to deal with
- * the Software without restriction, including without limitation the rights to
- * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
- * of the Software, and to permit persons to whom the Software is furnished to do
- * so, subject to the following conditions:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimers.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimers in the
- * documentation and/or other materials provided with the distribution.
- *
- * 3. Neither the names of Tresys Technology, nor the names of its contributors
- * may be used to endorse or promote products derived from this Software
- * without specific prior written permission.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
- * SOFTWARE.
- */
-
-package edu.illinois.ncsa.daffodil.processors.parsers
-
-import edu.illinois.ncsa.daffodil.processors.Evaluatable
-import edu.illinois.ncsa.daffodil.processors.PrimProcessor
-import edu.illinois.ncsa.daffodil.processors.TextProcessor
-import edu.illinois.ncsa.daffodil.processors.RuntimeData
-import edu.illinois.ncsa.daffodil.processors.TermRuntimeData
-
-/**
- * A PrimParser is a parser that contains no child parsers.
- * Combinators are NOT PrimParser
- *
- * This trait is preferred over PrimParserObject.
- */
-trait PrimParser
- extends PrimProcessor
- with Parser
-
-trait TextPrimParser
- extends PrimParser
- with TextProcessor
-
-// Deprecated and to be phased out. Use the trait PrimParser instead.
-abstract class PrimParserObject(override val context: RuntimeData)
- extends PrimParser {
- override def runtimeDependencies: Seq[Evaluatable[AnyRef]] = Nil
-}
-
-abstract class TextPrimParserObject(ctxt: TermRuntimeData)
- extends PrimParserObject(ctxt)
- with TextProcessor
-
-class NadaParser(override val context: RuntimeData) extends PrimParserObject(context) {
- override def toString = "Nada"
-
- override def parse(start: PState): Unit = {
- // do nothing
- }
-}
-
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PrimitivesTextNumber1.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PrimitivesTextNumber1.scala
index 756328f..2dcd036 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PrimitivesTextNumber1.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/PrimitivesTextNumber1.scala
@@ -65,7 +65,9 @@ case class ConvertTextCombinatorParser(
rd: TermRuntimeData,
valueParser: Parser,
converterParser: Parser)
- extends ParserObject(rd) {
+ extends CombinatorParser(rd) {
+
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Seq(valueParser, converterParser)
@@ -81,7 +83,9 @@ case class ConvertTextCombinatorParser(
case class ConvertTextNumberParser[S](
helper: ConvertTextNumberParserUnparserHelperBase[S],
nff: NumberFormatFactoryBase[S],
- e: ElementRuntimeData) extends TextPrimParserObject(e) {
+ override val context: ElementRuntimeData) extends TextPrimParser {
+ override lazy val runtimeDependencies = Nil
+
override def toString = "to(xs:" + helper.xsdType + ")"
def parse(start: PState): Unit = {
@@ -583,19 +587,19 @@ class NumberFormatFactoryStatic[S](context: ThrowsSDE,
val decSep =
if (decimalSepExpEv.isEmpty) Nope else One {
- val dse = decimalSepExpEv.get.optConstant.get
+ val dse = decimalSepExpEv.get.maybeConstant.get
getDecimalSepList(dse, context)
}
val groupSep =
if (groupingSepExpEv.isEmpty) Nope else One {
- val gse = groupingSepExpEv.get.optConstant.get
+ val gse = groupingSepExpEv.get.maybeConstant.get
getGroupingSep(gse, context)
}
val expRep = {
Assert.invariant(exponentRepExpEv.isConstant)
- getExponentRep(exponentRepExpEv.optConstant.get, context)
+ getExponentRep(exponentRepExpEv.maybeConstant.get, context)
}
val roundingInc: MaybeDouble = if (roundingIncrement.isEmpty) MaybeDouble.Nope else MaybeDouble { getRoundingIncrement(roundingIncrement.value, context) }
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/RepParsers.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/RepParsers.scala
index f5912ef..fd5c69e 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/RepParsers.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/RepParsers.scala
@@ -50,7 +50,9 @@ import edu.illinois.ncsa.daffodil.util.Numbers
import edu.illinois.ncsa.daffodil.processors.TermRuntimeData
abstract class RepParser(n: Long, rParser: Parser, context: ElementRuntimeData, baseName: String)
- extends ParserObject(context) {
+ extends CombinatorParser(context) {
+
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Seq(rParser)
@@ -371,8 +373,10 @@ class RepUnboundedParser(occursCountKind: OccursCountKind.Value, rParser: Parser
}
}
-class OccursCountExpressionParser(occursCountEv: Evaluatable[JLong], erd: ElementRuntimeData)
- extends ParserObject(erd) {
+class OccursCountExpressionParser(occursCountEv: Evaluatable[JLong],
+ override val context: ElementRuntimeData)
+ extends PrimParserNoData {
+ override lazy val runtimeDependencies = Nil
override lazy val childProcessors = Nil
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/SpecifiedLengthParsers.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/SpecifiedLengthParsers.scala
index e080046..cd443f4 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/SpecifiedLengthParsers.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/SpecifiedLengthParsers.scala
@@ -45,10 +45,12 @@ import edu.illinois.ncsa.daffodil.equality._
import java.lang.{ Long => JLong }
sealed abstract class SpecifiedLengthParserBase(eParser: Parser,
- erd: ElementRuntimeData)
- extends ParserObject(erd)
+ erd: ElementRuntimeData)
+ extends CombinatorParser(erd)
with CaptureParsingValueLength {
+ override lazy val runtimeDependencies = Nil
+
override lazy val childProcessors = Seq(eParser)
override def charsetEv = Assert.invariantFailed("Specified Length parsers should not capture value length using the charset")
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/UnorderedSequenceParser.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/UnorderedSequenceParser.scala
index b6dcfe6..d1f6f69 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/UnorderedSequenceParser.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/parsers/UnorderedSequenceParser.scala
@@ -39,7 +39,9 @@ class UnorderedSequenceParser(
sortOrder: Seq[(String, org.jdom2.Namespace)],
scalarMembers: Seq[(String, String, org.jdom2.Namespace)],
uoSeqParser: Parser)
- extends ParserObject(context) {
+ extends CombinatorParser(context) {
+
+ override lazy val runtimeDependencies = Nil
override def nom = "UnorderedSequence"
override lazy val childProcessors = Seq(uoSeqParser)
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/RepUnparsers.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/RepUnparsers.scala
index 7ebff86..bb9dca8 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/RepUnparsers.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/RepUnparsers.scala
@@ -38,7 +38,7 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.OccursCountKind
import edu.illinois.ncsa.daffodil.processors.TermRuntimeData
abstract class RepUnparser(n: Long, rUnparser: Unparser, context: ElementRuntimeData, baseName: String)
- extends UnparserObject(context) {
+ extends CombinatorUnparser(context) {
override lazy val childProcessors = Seq(rUnparser)
@@ -89,6 +89,8 @@ object Rep {
class RepExactlyNUnparser(n: Long, rUnparser: Unparser, context: ElementRuntimeData)
extends RepUnparser(n, rUnparser, context, "ExactlyN") {
+ override lazy val runtimeDependencies = Nil
+
def unparseAllRepeats(ustate: UState) {
1 to intN foreach { _ =>
{
@@ -104,6 +106,8 @@ class RepExactlyNUnparser(n: Long, rUnparser: Unparser, context: ElementRuntimeD
class RepAtMostTotalNUnparser(n: Long, rUnparser: Unparser, erd: ElementRuntimeData)
extends RepUnparser(n, rUnparser, erd, "AtMostTotalN") {
+ override lazy val runtimeDependencies = Nil
+
def unparseAllRepeats(ustate: UState): Unit = {
while (ustate.arrayPos <= intN && !ustate.isInspectArrayEnd) {
// Debugger.beforeRepetition(ustate, this)
@@ -118,6 +122,8 @@ class RepAtMostTotalNUnparser(n: Long, rUnparser: Unparser, erd: ElementRuntimeD
class RepExactlyTotalNUnparser(n: Long, rUnparser: Unparser, context: ElementRuntimeData)
extends RepUnparser(n, rUnparser, context, "ExactlyTotalN") {
+ override lazy val runtimeDependencies = Nil
+
def unparseAllRepeats(ustate: UState) {
Rep.loopExactlyTotalN(intN, rUnparser, ustate, context, this)
}
@@ -137,6 +143,8 @@ class RepExactlyTotalNUnparser(n: Long, rUnparser: Unparser, context: ElementRun
class RepUnboundedUnparser(occursCountKind: OccursCountKind.Value, rUnparser: Unparser, erd: ElementRuntimeData)
extends RepUnparser(-1, rUnparser, erd, "Unbounded") {
+ override lazy val runtimeDependencies = Nil
+
def unparseAllRepeats(ustate: UState) {
Assert.invariant(ustate.currentInfosetNodeMaybe.isDefined)
while (!ustate.isInspectArrayEnd) {
@@ -155,6 +163,9 @@ class RepUnboundedUnparser(occursCountKind: OccursCountKind.Value, rUnparser: Un
class RepAtMostOccursCountUnparser(rUnparser: Unparser, intN: Long, erd: ElementRuntimeData)
extends RepUnparser(intN, rUnparser, erd, "AtMostOccursCount") {
+
+ override lazy val runtimeDependencies = Nil
+
def unparseAllRepeats(ustate: UState) {
// repeat either n times, or occursCount times if that's less than n.
// val n = math.min(ustate.occursBounds, erd.minOccurs.get)
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/UState.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/UState.scala
index e341f69..c422261 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/UState.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/UState.scala
@@ -33,13 +33,8 @@
package edu.illinois.ncsa.daffodil.processors.unparsers
import java.io.ByteArrayOutputStream
-import java.nio.charset.Charset
-import java.nio.charset.CharsetDecoder
-import java.nio.charset.CharsetEncoder
-
import scala.Left
import scala.collection.mutable
-
import edu.illinois.ncsa.daffodil.api.DFDL
import edu.illinois.ncsa.daffodil.api.DataLocation
import edu.illinois.ncsa.daffodil.api.Diagnostic
@@ -79,6 +74,9 @@ import edu.illinois.ncsa.daffodil.processors.ParseOrUnparseState
import edu.illinois.ncsa.daffodil.api.DaffodilTunables
import edu.illinois.ncsa.daffodil.processors.NonTermRuntimeData
import edu.illinois.ncsa.daffodil.processors.TermRuntimeData
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharsetDecoder
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharsetEncoder
+import edu.illinois.ncsa.daffodil.processors.charset.BitsCharset
object ENoWarn { EqualitySuppressUnusedImportWarning() }
@@ -184,6 +182,10 @@ abstract class UState(
diagnostics = ue :: diagnostics
_processorStatus = new Failure(ue)
}
+
+ final override protected def checkBitOrder(): Unit = {
+ UnparserBitOrderChecks.checkUnparseBitOrder(this)
+ }
}
/**
@@ -214,8 +216,8 @@ class UStateForSuspension(
private def die = Assert.invariantFailed("Function should never be needed in UStateForSuspension")
- override def getDecoder(cs: Charset): CharsetDecoder = mainUState.getDecoder(cs)
- override def getEncoder(cs: Charset): CharsetEncoder = mainUState.getEncoder(cs)
+ override def getDecoder(cs: BitsCharset): BitsCharsetDecoder = mainUState.getDecoder(cs)
+ override def getEncoder(cs: BitsCharset): BitsCharsetEncoder = mainUState.getEncoder(cs)
// override def charBufferDataOutputStream = mainUState.charBufferDataOutputStream
override def withUnparserDataInputStream = mainUState.withUnparserDataInputStream
diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/Unparser.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/Unparser.scala
index 49f8f26..0f43eb2 100644
--- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/Unparser.scala
+++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/unparsers/Unparser.scala
@@ -41,12 +41,7 @@ import edu.illinois.ncsa.daffodil.processors.PrimProcessor
import edu.illinois.ncsa.daffodil.processors.ToBriefXMLImpl
import edu.illinois.ncsa.daffodil.processors.Processor
-/**
- * Vast majority of unparsers are actually Term unparsers. (The ones that are not are expression related e.g., SetVar)
- */
-abstract class TermUnparser(val termRuntimeData: TermRuntimeData) extends UnparserObject(termRuntimeData)
-
-trait Unparser
+sealed trait Unparser
extends Processor {
def isEmpty = false
@@ -67,14 +62,11 @@ trait Unparser
//
// ?? TODO: Should this be after the split below ??
if (ustate.dataProc.isDefined) ustate.dataProc.get.before(ustate, this)
-
- ustate.processor.context match {
- case mgrd: ModelGroupRuntimeData =>
- UnparserBitOrderChecks.checkUnparseBitOrder(ustate)
- case _ => // ok. Elements are checked elsewhere.
+ try {
+ unparse(ustate)
+ } finally {
+ ustate.resetFormatInfoCaches()
}
-
- unparse(ustate)
if (ustate.dataProc.isDefined) ustate.dataProc.get.after(ustate, this)
ustate.setMaybeProcessor(savedProc)
}
@@ -99,32 +91,41 @@ trait PrimUnparser
extends Unparser
with PrimProcessor
+/**
+ * An unparser that is primitive (no sub-unparsers), but doesn't write anything
+ * to a data stream (buffered or real), so alignment, bitOrder, etc. cannot
+ * apply to it.
+ */
+trait PrimUnparserNoData
+ extends Unparser
+ with PrimProcessorNoData
+
+/**
+ * Text primitive unparsers - primitive and textual only.
+ */
trait TextPrimUnparser
extends PrimUnparser
with TextProcessor
-// Deprecated and to be phased out. Use the trait Unparser instead.
-abstract class UnparserObject(override val context: RuntimeData)
- extends Unparser {
+abstract class CombinatorUnparser(override val context: RuntimeData)
+ extends Unparser with CombinatorProcessor
- override lazy val runtimeDependencies: Seq[Evaluatable[AnyRef]] = Nil
+trait SuspendableUnparser
+ extends PrimUnparser {
-}
+ protected def suspendableOperation: SuspendableOperation
-// Deprecated and to be phased out. Use the trait PrimUnparser instead.
-abstract class PrimUnparserObject(override val context: TermRuntimeData)
- extends PrimUnparser {
- override def runtimeDependencies: Seq[Evaluatable[AnyRef]] = Nil
+ override final def unparse(state: UState): Unit = {
+ suspendableOperation.run(state)
+ }
}
-abstract class TextPrimUnparserObject(ctxt: TermRuntimeData)
- extends PrimUnparserObject(ctxt)
- with TextProcessor
-
// No-op, in case an optimization lets one of these sneak thru.
// TODO: make this fail, and test optimizer sufficiently to know these
// do NOT get through.
-class EmptyGramUnparser(context: TermRuntimeData = null) extends UnparserObject(context) {
+final class EmptyGramUnparser(override val context: TermRuntimeData = null) extends PrimUnparserNoData {
+
+ override lazy val runtimeDependencies = Nil
def unparse(ustate: UState) {
Assert.invariantFailed("EmptyGramUnparsers are all supposed to optimize out!")
@@ -138,7 +139,10 @@ class EmptyGramUnparser(context: TermRuntimeData = null) extends UnparserObject(
override def toString = toBriefXML()
}
-class ErrorUnparser(context: TermRuntimeData = null) extends UnparserObject(context) {
+final class ErrorUnparser(override val context: TermRuntimeData = null) extends PrimUnparserNoData {
+
+ override lazy val runtimeDependencies = Nil
+
def unparse(ustate: UState) {
Assert.abort("Error Unparser")
}
@@ -148,10 +152,12 @@ class ErrorUnparser(context: TermRuntimeData = null) extends UnparserObject(cont
override def toString = "Error Unparser"
}
-class SeqCompUnparser(context: RuntimeData, val childUnparsers: Array[Unparser])
- extends UnparserObject(context)
+final class SeqCompUnparser(context: RuntimeData, val childUnparsers: Array[Unparser])
+ extends CombinatorUnparser(context)
with ToBriefXMLImpl {
+ override lazy val runtimeDependencies = Nil
+
override val childProcessors = childUnparsers.toSeq
override def nom = "seq"
@@ -171,7 +177,11 @@ class SeqCompUnparser(context: RuntimeData, val childUnparsers: Array[Unparser])
}
}
-case class DummyUnparser(primitiveName: String) extends UnparserObject(null) {
+case class DummyUnparser(primitiveName: String) extends PrimUnparserNoData {
+
+ override def context = null
+
+ override lazy val runtimeDependencies = Nil
override def isEmpty = true
@@ -182,7 +192,7 @@ case class DummyUnparser(primitiveName: String) extends UnparserObject(null) {
override def toString = "DummyUnparser (%s)".format(primitiveName)
}
-case class NotUnparsableUnparser(context: ElementRuntimeData) extends Unparser {
+case class NotUnparsableUnparser(override val context: ElementRuntimeData) extends PrimUnparserNoData {
def unparse(state: UState): Unit = {
// We can't use state.SDE because that needs the infoset to determine the
diff --git a/daffodil-tdml/src/main/scala/edu/illinois/ncsa/daffodil/tdml/TDMLRunner.scala b/daffodil-tdml/src/main/scala/edu/illinois/ncsa/daffodil/tdml/TDMLRunner.scala
index 7a74132..b22b462 100644
--- a/daffodil-tdml/src/main/scala/edu/illinois/ncsa/daffodil/tdml/TDMLRunner.scala
+++ b/daffodil-tdml/src/main/scala/edu/illinois/ncsa/daffodil/tdml/TDMLRunner.scala
@@ -75,7 +75,6 @@ import java.nio.CharBuffer
import java.nio.channels.Channels
import java.nio.charset.CoderResult
import java.io.ByteArrayInputStream
-import edu.illinois.ncsa.daffodil.io.NonByteSizeCharsetEncoder
import scala.language.postfixOps
import java.nio.file.Paths
import java.nio.file.Files
@@ -86,12 +85,14 @@ import edu.illinois.ncsa.daffodil.processors.UnparseResult
import edu.illinois.ncsa.daffodil.cookers.EntityReplacer
import edu.illinois.ncsa.daffodil.configuration.ConfigurationLoader
import edu.illinois.ncsa.daffodil.dsom.ExpressionCompilers
-import edu.illinois.ncsa.daffodil.io.NonByteSizeCharset
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.BitOrder
import edu.illinois.ncsa.daffodil.infoset._
import edu.illinois.ncsa.daffodil.util.MaybeBoolean
import edu.illinois.ncsa.daffodil.dpath.NodeInfo
import edu.illinois.ncsa.daffodil.api.DaffodilTunables
+import edu.illinois.ncsa.daffodil.processors.charset.NBitsWidth_BitsCharset
+import edu.illinois.ncsa.daffodil.processors.charset.NBitsWidth_BitsCharsetEncoder
+import java.nio.charset.{ Charset => JavaCharset }
/**
* Parses and runs tests expressed in IBM's contributed tdml "Test Data Markup Language"
@@ -1185,7 +1186,7 @@ object VerifyTestCase {
}
}
- private val cs8859 = java.nio.charset.Charset.forName("iso-8859-1")
+ private val cs8859 = JavaCharset.forName("iso-8859-1")
def verifyBinaryOrMixedData(expectedData: DFDL.Input, actualOutStream: java.io.ByteArrayOutputStream) {
val actualBytes = actualOutStream.toByteArray
@@ -1498,7 +1499,7 @@ class TextDocumentPart(part: Node, parent: Document) extends DataDocumentPart(pa
val upperName = encodingName.toUpperCase
val cs = CharsetUtils.getCharset(upperName)
cs match {
- case bitEnc: NonByteSizeCharset => {
+ case bitEnc: NBitsWidth_BitsCharset => {
(bitEnc.requiredBitOrder, partBitOrder) match {
case (BitOrder.LeastSignificantBitFirst, LSBFirst) => //ok
case (BitOrder.MostSignificantBitFirst, MSBFirst) => //ok
@@ -1547,8 +1548,8 @@ class TextDocumentPart(part: Node, parent: Document) extends DataDocumentPart(pa
bb.flip()
val res = (0 to bb.limit() - 1).map { bb.get(_) }
// val bitsAsString = bytes2Bits(res.toArray)
- val enc = encoder.asInstanceOf[NonByteSizeCharsetEncoder]
- val nBits = s.length * enc.bitWidthOfACodeUnit
+ val enc = encoder.asInstanceOf[NBitsWidth_BitsCharsetEncoder]
+ val nBits = s.length * enc.bitsCharset.bitWidthOfACodeUnit
val bitStrings = res.map { b => (b & 0xFF).toBinaryString.reverse.padTo(8, '0').reverse }.toList
val allBits = bitStrings.reverse.mkString.takeRight(nBits)
val bitChunks = allBits.reverse.sliding(byteSize, byteSize).map { _.reverse }.toList
@@ -1571,8 +1572,8 @@ class TextDocumentPart(part: Node, parent: Document) extends DataDocumentPart(pa
lazy val dataBits = {
val bytesAsStrings =
- encoder.charset() match {
- case nbs: NonByteSizeCharset =>
+ encoder.bitsCharset match {
+ case nbs: NBitsWidth_BitsCharset =>
encodeWithNonByteSizeEncoder(textContentWithoutEntities, nbs.bitWidthOfACodeUnit)
case _ =>
encodeWith8BitEncoder(textContentWithoutEntities)
diff --git a/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section05/simple_types/BitOrder.tdml b/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section05/simple_types/BitOrder.tdml
index c664e3a..cafcc40 100644
--- a/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section05/simple_types/BitOrder.tdml
+++ b/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section05/simple_types/BitOrder.tdml
@@ -42,7 +42,8 @@
<tdml:defineSchema name="s">
<dfdl:format ref="ex:daffodilTest1" representation="binary"
lengthUnits="bits" bitOrder='leastSignificantBitFirst' lengthKind='explicit'
- byteOrder='littleEndian' alignmentUnits='bits' alignment='1'/>
+ byteOrder='littleEndian' alignmentUnits='bits' alignment='1'
+ encoding="X-DFDL-US-ASCII-7-BIT-PACKED"/>
<xs:element name="tabBI" dfdl:lengthKind='implicit' >
<xs:complexType>
@@ -57,7 +58,7 @@
<xs:element name="URN" type="xs:unsignedInt" dfdl:length="24" />
<xs:element name="FPI_UnitName" type="xs:unsignedInt"
dfdl:length="1" />
- <xs:element name="UnitName" type="xs:string" dfdl:encoding="X-DFDL-US-ASCII-7-BIT-PACKED"
+ <xs:element name="UnitName" type="xs:string"
dfdl:lengthKind="delimited" dfdl:terminator="" />
<xs:element name="GPI_RecipAddrGroup" type="xs:unsignedInt"
dfdl:length="1" />
diff --git a/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section05/simple_types/BitOrderInvalid.tdml b/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section05/simple_types/BitOrderInvalid.tdml
index 5b4f9af..fd91b49 100644
--- a/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section05/simple_types/BitOrderInvalid.tdml
+++ b/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section05/simple_types/BitOrderInvalid.tdml
@@ -68,7 +68,7 @@
<documentPart type="bits" bitOrder="MSBFirst" >011 01010100</documentPart>
</document>
<tdml:errors>
- <tdml:error></tdml:error>
+ <tdml:error>bitOrder</tdml:error>
</tdml:errors>
</tdml:parserTestCase>
diff --git a/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section11/content_framing_properties/ContentFramingProps.tdml b/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section11/content_framing_properties/ContentFramingProps.tdml
index e5d10c9..0312673 100644
--- a/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section11/content_framing_properties/ContentFramingProps.tdml
+++ b/daffodil-test/src/test/resources/edu/illinois/ncsa/daffodil/section11/content_framing_properties/ContentFramingProps.tdml
@@ -80,9 +80,9 @@
<xs:element name="e1" type="xs:string" dfdl:lengthKind="delimited"
dfdl:terminator=";" />
<xs:element name="e2" type="xs:string" dfdl:lengthKind="explicit"
- dfdl:length="2" dfdl:encoding="{ ../ex:e1 }" dfdl:bitOrder="leastSignificantBitFirst" dfdl:byteOrder="littleEndian" />
+ dfdl:length="2" dfdl:encoding="{ ../ex:e1 }" dfdl:bitOrder="mostSignificantBitFirst" dfdl:byteOrder="littleEndian" />
<xs:element name="e3" type="xs:unsignedInt" dfdl:lengthKind="explicit" dfdl:length="4" dfdl:lengthUnits="bits"
- dfdl:bitOrder="leastSignificantBitFirst" dfdl:byteOrder="littleEndian"/>
+ dfdl:byteOrder="littleEndian"/>
</xs:sequence>
</xs:complexType>
</xs:element>
@@ -1000,7 +1000,23 @@
<xs:complexType>
<xs:sequence>
<xs:element name="string7Bit" type="xs:string" dfdl:length="1" dfdl:lengthKind="explicit" />
- <xs:element name="string8Bit" type="xs:string" dfdl:length="1" dfdl:lengthKind="explicit" dfdl:encoding="US-ASCII" dfdl:alignment="8" />
+
+ <xs:sequence dfdl:alignment="8"/><!-- align to 8 bit boundary so we can change bit order -->
+
+ <!--
+ Interesting. The above element leaves us at bit 7, and bitOrder LSBF.
+ But the next thing below has bitOrder MSBF and encoding US-ASCII which has mandatory bit
+ alignment of 8.
+
+ So we should automatically move over to a byte boundary (the mandatory byte alignment)
+ but we do so filling with the fill byte according to the bitOrder of the preceding thing.
+
+ Daffodil doesn't do this currently. Bug DAFFODIL-1888
+ -->
+ <xs:element name="string8Bit" type="xs:string" dfdl:length="1" dfdl:lengthKind="explicit"
+ dfdl:encoding="US-ASCII"
+ dfdl:bitOrder="mostSignificantBitFirst"
+ dfdl:alignment="8" />
</xs:sequence>
</xs:complexType>
</xs:element>
@@ -1250,8 +1266,18 @@
<xs:element name="e3" type="xs:string" dfdl:lengthKind="explicit"
dfdl:length="9" />
- <xs:element name="e4" type="xs:string" dfdl:lengthKind="explicit"
- dfdl:length="2" dfdl:bitOrder="mostSignificantBitFirst" />
+ <xs:element name="e4">
+ <xs:complexType>
+ <xs:sequence>
+ <!-- single bit field leaves us after first bit, and in MSBF bit order -->
+ <xs:element name="bit" type="xs:int" dfdl:lengthKind="explicit" dfdl:length="1"
+ dfdl:representation="binary" dfdl:lengthUnits="bits" dfdl:bitOrder="mostSignificantBitFirst"/>
+ <!-- this encoding can go on any bit boundary, but requires LSBF -->
+ <xs:element name="s" type="xs:string" dfdl:lengthKind="explicit"
+ dfdl:length="2" dfdl:alignmentUnits="bits"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
<xs:element name="e6" type="xs:string" dfdl:lengthKind="delimited"
dfdl:terminator="" dfdl:representation="text" dfdl:alignmentUnits="bits" dfdl:alignment="4" />
@@ -1378,11 +1404,12 @@
model="packed5Bit">
<tdml:document bitOrder="LSBFirst">
<tdml:documentPart type="bits" byteOrder="RTL"><![CDATA[
- 00000 11101
+ 00000 11101 1
]]></tdml:documentPart>
</tdml:document>
<tdml:errors>
- <tdml:error>requires bit order 'leastSignificantBitFirst'</tdml:error>
+ <tdml:error>bitOrder on a byte boundary</tdml:error>
+ <tdml:error>2</tdml:error>
</tdml:errors>
</tdml:parserTestCase>
diff --git a/daffodil-test/src/test/scala-debug/edu/illinois/ncsa/daffodil/section05/simple_types/TestSimpleTypesDebug.scala b/daffodil-test/src/test/scala-debug/edu/illinois/ncsa/daffodil/section05/simple_types/TestSimpleTypesDebug.scala
index fb10662..e9c3362 100644
--- a/daffodil-test/src/test/scala-debug/edu/illinois/ncsa/daffodil/section05/simple_types/TestSimpleTypesDebug.scala
+++ b/daffodil-test/src/test/scala-debug/edu/illinois/ncsa/daffodil/section05/simple_types/TestSimpleTypesDebug.scala
@@ -51,6 +51,7 @@ object TestSimpleTypesDebug {
val runner1 = Runner(testDir, "BitOrder.tdml")
val runnerST = Runner(testDir, "simple-type-bases.tdml")
val runner_01 = Runner(testDir_01, "dpaext1.tdml")
+ val runner3 = Runner(testDir, "BitOrderInvalid.tdml")
@AfterClass def shutDown() {
runner.reset
@@ -115,11 +116,10 @@ class TestSimpleTypesDebug {
@Test def test_whiteSpaceAfterLax() { runner2.runOneTest("whiteSpaceAfterLax") }
@Test def test_redefinedFormat() { runner2.runOneTest("redefinedFormat") }
- @Test def test_bigEndianLeastFirst() { runner1.runOneTest("bigEndianLeastFirst") }
@Test def test_bitOrderChange() { runner1.runOneTest("bitOrderChange") }
@Test def test_bitOrderDocument() { runner1.runOneTest("bitOrderDocument") }
@Test def test_bitOrderTypeByte() { runner1.runOneTest("bitOrderTypeByte") }
@Test def test_bitOrderChangeInvalid3() { runner1.runOneTest("bitOrderChangeInvalid3") }
- @Test def test_bitOrderChangeInvalid() { runner1.runOneTest("bitOrderChangeInvalid") }
+ @Test def test_bitOrderChangeInvalid() { runner3.runOneTest("bitOrderChangeInvalid") }
}
diff --git a/daffodil-test/src/test/scala/edu/illinois/ncsa/daffodil/section05/simple_types/TestSimpleTypesNew.scala b/daffodil-test/src/test/scala/edu/illinois/ncsa/daffodil/section05/simple_types/TestSimpleTypesNew.scala
index 2172710..a36a953 100644
--- a/daffodil-test/src/test/scala/edu/illinois/ncsa/daffodil/section05/simple_types/TestSimpleTypesNew.scala
+++ b/daffodil-test/src/test/scala/edu/illinois/ncsa/daffodil/section05/simple_types/TestSimpleTypesNew.scala
@@ -40,9 +40,10 @@ object TestSimpleTypesNew {
private val testDir = "/edu/illinois/ncsa/daffodil/section05/simple_types/"
val runner = Runner(testDir, "SimpleTypes.tdml")
-
+ val runner1 = Runner(testDir, "BitOrder.tdml")
@AfterClass def shutdown {
runner.reset
+ runner1.reset
}
}
@@ -57,4 +58,7 @@ class TestSimpleTypesNew {
@Test def test_dateTime_calendarTimeZone_EST() { runner.runOneTest("dateTime_calendarTimeZone_EST") }
@Test def test_hexBinary_specifiedLengthUnaligned() { runner.runOneTest("hexBinary_specifiedLengthUnaligned") }
+
+ // DAFFODIL-1001 fixed.
+ @Test def test_bigEndianLeastFirst() { runner1.runOneTest("bigEndianLeastFirst") }
}
--
To stop receiving notification emails like this one, please contact
mbeckerle@apache.org.