You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by sl...@apache.org on 2018/07/17 20:33:19 UTC
[incubator-daffodil] branch master updated: Added a layer transform
that swaps bytes within an array of 32 bit words
This is an automated email from the ASF dual-hosted git repository.
slawrence pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git
The following commit(s) were added to refs/heads/master by this push:
new ff2d3fb Added a layer transform that swaps bytes within an array of 32 bit words
ff2d3fb is described below
commit ff2d3fb4144b99f2ff24e3dbd01608f8b02accef
Author: Marshall Miller <ma...@quarksecurity.com>
AuthorDate: Mon May 14 18:28:04 2018 -0400
Added a layer transform that swaps bytes within an array of 32 bit words
This is a layer transform that allows Daffodil to support parsing
Link-16 J messages embedded within DIS PDUs, commonly referred to as SISO-J.
DAFFODIL-1888
---
.../org/apache/daffodil/layers/TestLayers.scala | 50 +++++
.../apache/daffodil/xsd/DFDL_part1_simpletypes.xsd | 7 +
.../daffodil/layers/ByteSwapTransformer.scala | 224 +++++++++++++++++++++
.../apache/daffodil/layers/LayerTransformer.scala | 1 +
.../daffodil/layers/TestByteSwapStream.scala | 74 +++++++
5 files changed, 356 insertions(+)
diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/layers/TestLayers.scala b/daffodil-core/src/test/scala/org/apache/daffodil/layers/TestLayers.scala
index a36af08..0bb0c03 100644
--- a/daffodil-core/src/test/scala/org/apache/daffodil/layers/TestLayers.scala
+++ b/daffodil-core/src/test/scala/org/apache/daffodil/layers/TestLayers.scala
@@ -358,4 +358,54 @@ a few lines of pointless text like this.""".replace("\n", " ")
val areTracing = false
TestUtils.testUnparsing(sch, infoset, data, areTracing)
}
+
+ val le32BitData = Array[Byte](0x01, // BE MSBF
+ 0x43, 0x33, 0x33, 0x32, // fourbyteswap + LE LSBF (parsed right to left four bytes at a time)
+ 0x55, 0x54, // fourbyteswap + LE LSBF
+ 0x67) // BE MSBF
+
+ val le32BitSchema =
+ SchemaUtils.dfdlTestSchema(
+ <dfdl:format ref="tns:GeneralFormat" bitOrder="leastSignificantBitFirst"
+ byteOrder="littleEndian" alignmentUnits="bits" alignment="1"
+ lengthKind="explicit" lengthUnits="bits"/>,
+ <xs:element name="e1" dfdl:lengthKind="implicit">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="s0" type="xs:hexBinary" dfdl:length="4" dfdl:byteOrder="bigEndian" dfdl:bitOrder="mostSignificantBitFirst"/>
+ <xs:element name="s1" type="xs:hexBinary" dfdl:length="4" dfdl:byteOrder="bigEndian" dfdl:bitOrder="mostSignificantBitFirst"/>
+ <xs:sequence dfdl:layerTransform="fourbyteswap" dfdl:layerLengthKind="explicit" dfdl:layerLengthUnits="bytes" dfdl:layerLength="6">
+ <xs:sequence>
+ <xs:element name="s2" type="xs:hexBinary" dfdl:length="4"/>
+ <xs:element name="s3" type="xs:hexBinary" dfdl:length="24"/>
+ <xs:element name="s4" type="xs:hexBinary" dfdl:length="8"/>
+ <xs:element name="s5" type="xs:hexBinary" dfdl:length="12"/>
+ </xs:sequence>
+ </xs:sequence>
+ <xs:element name="s6" type="xs:hexBinary" dfdl:length="4" dfdl:byteOrder="bigEndian" dfdl:bitOrder="mostSignificantBitFirst"/>
+ <xs:element name="s7" type="xs:hexBinary" dfdl:length="4" dfdl:byteOrder="bigEndian" dfdl:bitOrder="mostSignificantBitFirst"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>, elementFormDefault = "qualified")
+
+ @Test def testFourByteSwapLayer() {
+ val sch = le32BitSchema
+ val data = le32BitData
+ val infoset =
+ <e1 xmlns={ example }>
+ <s0>00</s0>
+ <s1>01</s1>
+ <s2>02</s2>
+ <s3>333333</s3>
+ <s4>44</s4>
+ <s5>0555</s5>
+ <s6>06</s6>
+ <s7>07</s7>
+ </e1>
+
+ val (_, actual) = TestUtils.testBinary(sch, data, areTracing = false)
+ TestUtils.assertEqualsXMLElements(infoset, actual)
+
+ TestUtils.testUnparsingBinary(sch, infoset, data)
+ }
}
diff --git a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd
index 280c367..cdc1f02 100644
--- a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd
+++ b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd
@@ -685,6 +685,13 @@
<xsd:simpleType name="LayerTransformEnum">
<xsd:restriction base="xsd:token">
+ <xsd:enumeration value="fourbyteswap">
+ <xsd:annotation>
+ <xsd:documentation>
+ Swap bytes in 32 bit words.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:enumeration>
<xsd:enumeration value="base64_MIME">
<xsd:annotation>
<xsd:documentation>
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/ByteSwapTransformer.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/ByteSwapTransformer.scala
new file mode 100644
index 0000000..858510c
--- /dev/null
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/ByteSwapTransformer.scala
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.layers
+
+import java.io.OutputStream
+import java.io.InputStream
+import java.util.ArrayDeque
+import java.util.Deque
+
+import org.apache.daffodil.exceptions.Assert
+import org.apache.daffodil.schema.annotation.props.gen.LayerLengthKind
+import org.apache.daffodil.schema.annotation.props.gen.LayerLengthUnits
+import org.apache.daffodil.util.Maybe
+import org.apache.daffodil.processors.TermRuntimeData
+import org.apache.daffodil.processors.LayerLengthInBytesEv
+import org.apache.daffodil.processors.LayerBoundaryMarkEv
+import org.apache.daffodil.processors.LayerCharsetEv
+import org.apache.daffodil.processors.parsers.PState
+import org.apache.daffodil.io.ExplicitLengthLimitingStream
+import org.apache.daffodil.processors.unparsers.UState
+
+/**
+ * An input stream wrapper that re-orders bytes according to wordsize.
+ *
+ * Bytes within the wrapped input stream are re-ordered wordsize bytes at a time.
+ * For example, if the wrapped input stream contains 10 bytes and wordsize is
+ * 4, then the bytes from the wrapped input stream are returned in the
+ * order 4 3 2 1 8 7 6 5 10 9. If wordsize were 2 then the bytes from the
+ * wrapped input stream are returned in the order 2 1 4 3 6 5 8 7 10 9.
+ */
+class ByteSwapInputStream(wordsize: Int, jis: InputStream)
+ extends InputStream {
+
+ object State extends org.apache.daffodil.util.Enum {
+ abstract sealed trait Type extends EnumValueType
+
+ /**
+ * Buffering bytes in a word.
+ */
+ case object Filling extends Type
+
+ /**
+ * Return buffered bytes in a word in LIFO order.
+ */
+ case object Emptying extends Type
+
+ /**
+ * No more data. Drain buffered bytes.
+ */
+ case object Draining extends Type
+
+ /**
+ * No more data and no buffered bytes.
+ */
+ case object Done extends Type
+ }
+
+ private var c: Int = -2
+ private val stack: Deque[Int] = new ArrayDeque[Int](wordsize)
+ private var state: State.Type = State.Filling
+
+ /**
+ * Swap wordsize bytes at a time
+ *
+ */
+ override def read(): Int = {
+ import State._
+ if (state eq Done) return -1
+ while (state != Done) {
+ state match {
+ case Filling => {
+ c = jis.read()
+ if (c == -1) {
+ state = Draining
+ } else {
+ stack.push(c)
+ if (stack.size() == wordsize) {
+ state = Emptying
+ }
+ }
+ }
+ case Emptying => {
+ if (stack.isEmpty()) {
+ state = Filling
+ } else {
+ c = stack.pop()
+ return c
+ }
+ }
+ case Draining => {
+ if (stack.isEmpty()) {
+ state = Done
+ return -1
+ } else {
+ c = stack.pop()
+ return c
+ }
+ }
+ case Done =>
+ Assert.invariantFailed("Done state not allowed.")
+ }
+ }
+ Assert.invariantFailed("No fall through to here.")
+ }
+}
+
+/**
+ * An output stream wrapper that re-orders bytes according to wordsize.
+ *
+ * Bytes sent to the wrapped output stream are re-ordered wordsize bytes at a
+ * time. For example, if 10 bytes are written and wordsize is 4, then the
+ * bytes are written to the wrapped output stream in the
+ * order 4 3 2 1 8 7 6 5 10 9. If wordsize were 2 then the bytes are written
+ * to the wrapped output stream in the order 2 1 4 3 6 5 8 7 10 9.
+ */
+class ByteSwapOutputStream(wordsize: Int, jos: OutputStream)
+ extends OutputStream {
+
+ private val stack: Deque[Byte] = new ArrayDeque[Byte](wordsize)
+ private var closed = false
+
+ override def close(): Unit = {
+ if (!closed) {
+ while (!stack.isEmpty()) {
+ jos.write(stack.pop())
+ }
+ jos.close()
+ closed = true
+ }
+ }
+
+ override def write(bInt: Int): Unit = {
+ Assert.usage(!closed)
+ stack.push(bInt.toByte)
+ if (stack.size() == wordsize) {
+ while(!stack.isEmpty()) {
+ jos.write(stack.pop())
+ }
+ }
+ }
+}
+
+/**
+ * A LayerTransformer that re-orders bytes for the over/underlying layer.
+ */
+class ByteSwapTransformer(wordsize: Int, layerLengthInBytesEv: LayerLengthInBytesEv)
+ extends LayerTransformer() {
+
+ override def wrapLayerDecoder(jis: java.io.InputStream) = {
+ val s = new ByteSwapInputStream(wordsize, jis)
+ s
+ }
+
+ override def wrapLimitingStream(jis: java.io.InputStream, state: PState) = {
+ val layerLengthInBytes: Int = layerLengthInBytesEv.evaluate(state).toInt
+
+ val s = new ExplicitLengthLimitingStream(jis, layerLengthInBytes)
+ s
+ }
+
+ override protected def wrapLayerEncoder(jos: java.io.OutputStream): java.io.OutputStream = {
+ val s = new ByteSwapOutputStream(wordsize, jos)
+ s
+ }
+
+ override protected def wrapLimitingStream(jos: java.io.OutputStream, state: UState): java.io.OutputStream = {
+ jos // just return jos. The way the length will be used/stored is by way of
+ // taking the content length of the enclosing element. That will measure the
+ // length relative to the "ultimate" data output stream.
+ }
+}
+
+/**
+ * LayerTransformerFactory for ByteSwapTransformer instances.
+ *
+ * Requires that layerLengthKind="explicit".
+ */
+sealed abstract class ByteSwapTransformerFactory(wordsize: Int, name: String)
+ extends LayerTransformerFactory(name) {
+
+ override def newInstance(maybeLayerCharsetEv: Maybe[LayerCharsetEv],
+ maybeLayerLengthKind: Maybe[LayerLengthKind],
+ maybeLayerLengthInBytesEv: Maybe[LayerLengthInBytesEv],
+ maybeLayerLengthUnits: Maybe[LayerLengthUnits],
+ maybeLayerBoundaryMarkEv: Maybe[LayerBoundaryMarkEv],
+ trd: TermRuntimeData): LayerTransformer = {
+
+ trd.schemaDefinitionUnless(maybeLayerLengthKind.isDefined,
+ "The propert dfdl:layerLengthKind must be defined.")
+
+ val xformer =
+ maybeLayerLengthKind.get match {
+ case LayerLengthKind.Explicit => {
+ new ByteSwapTransformer(wordsize, maybeLayerLengthInBytesEv.get)
+ }
+ case x =>
+ trd.SDE("Property dfdl:layerLengthKind can only be 'explicit', but was '%s'",
+ x.toString)
+ }
+ xformer
+ }
+}
+
+/**
+ * Factory for ByteSwapTransformer instances with a wordsize of 4.
+ */
+object FourByteSwapTransformerFactory
+ extends ByteSwapTransformerFactory(4, "fourbyteswap")
+
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala
index 3687b5a..1e62410 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala
@@ -101,6 +101,7 @@ object LayerTransformerFactory {
register(IMFLineFoldedTransformerFactory)
register(ICalendarLineFoldedTransformerFactory)
register(AISPayloadArmoringTransformerFactory)
+ register(FourByteSwapTransformerFactory)
}
/**
diff --git a/daffodil-runtime1/src/test/scala/org/apache/daffodil/layers/TestByteSwapStream.scala b/daffodil-runtime1/src/test/scala/org/apache/daffodil/layers/TestByteSwapStream.scala
new file mode 100644
index 0000000..9fc279c
--- /dev/null
+++ b/daffodil-runtime1/src/test/scala/org/apache/daffodil/layers/TestByteSwapStream.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.layers
+
+import junit.framework.Assert._
+import java.io._
+import org.junit.Test
+import java.nio.charset.StandardCharsets
+
+class TestByteSwapStreams {
+
+ val iso8859 = StandardCharsets.ISO_8859_1
+
+ val unswapped32BitData = Array[Byte](0x76, 0x54, 0x32, 0x10,
+ 0xBA.toByte, 0x98.toByte)
+
+ val swapped32BitData = Array[Byte](0x10, 0x32, 0x54, 0x76, 0x98.toByte, 0xBA.toByte)
+
+ @Test def testFourByteSwapInputStream() = {
+ val data = unswapped32BitData
+ val bba = new ByteArrayInputStream(data)
+ val bss = new ByteSwapInputStream(4, bba)
+
+ val baos = new ByteArrayOutputStream()
+ var c: Int = -1
+ while ({
+ c = bss.read()
+ c != -1
+ }) {
+ baos.write(c)
+ }
+ baos.close()
+ val result = new String(baos.toByteArray(), iso8859)
+ val expected = new String(swapped32BitData, iso8859)
+ assertEquals(expected, result)
+ }
+
+ @Test def testFourByteSwapOutputStream() = {
+ val data = swapped32BitData
+ val bba = new ByteArrayInputStream(data)
+
+ val baos = new ByteArrayOutputStream()
+ val bsos = new ByteSwapOutputStream(4, baos)
+ var c: Int = -1
+ while ({
+ c = bba.read()
+ c != -1
+ }) {
+ bsos.write(c)
+ }
+ bsos.close()
+ baos.close()
+
+ val result = new String(baos.toByteArray(), iso8859)
+ val expected = new String(unswapped32BitData, iso8859)
+ assertEquals(expected, result)
+ }
+
+}