You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by sl...@apache.org on 2018/07/17 20:33:19 UTC

[incubator-daffodil] branch master updated: Added a layer transform that swaps bytes within an array of 32 bit words

This is an automated email from the ASF dual-hosted git repository.

slawrence pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git


The following commit(s) were added to refs/heads/master by this push:
     new ff2d3fb  Added a layer transform that swaps bytes within an array of 32 bit words
ff2d3fb is described below

commit ff2d3fb4144b99f2ff24e3dbd01608f8b02accef
Author: Marshall Miller <ma...@quarksecurity.com>
AuthorDate: Mon May 14 18:28:04 2018 -0400

    Added a layer transform that swaps bytes within an array of 32 bit words
    
    This is a layer transform that allows Daffodil to support parsing
    Link-16 J messages embedded within DIS PDUs, commonly referred to as SISO-J.
    
    DAFFODIL-1888
---
 .../org/apache/daffodil/layers/TestLayers.scala    |  50 +++++
 .../apache/daffodil/xsd/DFDL_part1_simpletypes.xsd |   7 +
 .../daffodil/layers/ByteSwapTransformer.scala      | 224 +++++++++++++++++++++
 .../apache/daffodil/layers/LayerTransformer.scala  |   1 +
 .../daffodil/layers/TestByteSwapStream.scala       |  74 +++++++
 5 files changed, 356 insertions(+)

diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/layers/TestLayers.scala b/daffodil-core/src/test/scala/org/apache/daffodil/layers/TestLayers.scala
index a36af08..0bb0c03 100644
--- a/daffodil-core/src/test/scala/org/apache/daffodil/layers/TestLayers.scala
+++ b/daffodil-core/src/test/scala/org/apache/daffodil/layers/TestLayers.scala
@@ -358,4 +358,54 @@ a few lines of pointless text like this.""".replace("\n", " ")
     val areTracing = false
     TestUtils.testUnparsing(sch, infoset, data, areTracing)
   }
+
+  val le32BitData = Array[Byte](0x01,                   // BE MSBF
+                                0x43, 0x33, 0x33, 0x32, // fourbyteswap + LE LSBF (parsed right to left four bytes at a time)
+                                            0x55, 0x54, // fourbyteswap + LE LSBF
+                                0x67)                   // BE MSBF
+
+  val le32BitSchema =
+    SchemaUtils.dfdlTestSchema(
+      <dfdl:format ref="tns:GeneralFormat" bitOrder="leastSignificantBitFirst"
+                   byteOrder="littleEndian" alignmentUnits="bits" alignment="1"
+                   lengthKind="explicit" lengthUnits="bits"/>,
+      <xs:element name="e1" dfdl:lengthKind="implicit">
+        <xs:complexType>
+          <xs:sequence>
+            <xs:element name="s0" type="xs:hexBinary" dfdl:length="4" dfdl:byteOrder="bigEndian" dfdl:bitOrder="mostSignificantBitFirst"/>
+            <xs:element name="s1" type="xs:hexBinary" dfdl:length="4" dfdl:byteOrder="bigEndian" dfdl:bitOrder="mostSignificantBitFirst"/>
+            <xs:sequence dfdl:layerTransform="fourbyteswap" dfdl:layerLengthKind="explicit" dfdl:layerLengthUnits="bytes" dfdl:layerLength="6">
+              <xs:sequence>
+                <xs:element name="s2" type="xs:hexBinary" dfdl:length="4"/>
+                <xs:element name="s3" type="xs:hexBinary" dfdl:length="24"/>
+                <xs:element name="s4" type="xs:hexBinary" dfdl:length="8"/>
+                <xs:element name="s5" type="xs:hexBinary" dfdl:length="12"/>
+              </xs:sequence>
+            </xs:sequence>
+            <xs:element name="s6" type="xs:hexBinary" dfdl:length="4" dfdl:byteOrder="bigEndian" dfdl:bitOrder="mostSignificantBitFirst"/>
+            <xs:element name="s7" type="xs:hexBinary" dfdl:length="4" dfdl:byteOrder="bigEndian" dfdl:bitOrder="mostSignificantBitFirst"/>
+          </xs:sequence>
+        </xs:complexType>
+      </xs:element>, elementFormDefault = "qualified")
+
+  @Test def testFourByteSwapLayer() {
+    val sch = le32BitSchema
+    val data = le32BitData
+    val infoset =
+      <e1 xmlns={ example }>
+        <s0>00</s0>
+        <s1>01</s1>
+        <s2>02</s2>
+        <s3>333333</s3>
+        <s4>44</s4>
+        <s5>0555</s5>
+        <s6>06</s6>
+        <s7>07</s7>
+      </e1>
+
+    val (_, actual) = TestUtils.testBinary(sch, data, areTracing = false)
+    TestUtils.assertEqualsXMLElements(infoset, actual)
+
+    TestUtils.testUnparsingBinary(sch, infoset, data)
+  }
 }
diff --git a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd
index 280c367..cdc1f02 100644
--- a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd
+++ b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd
@@ -685,6 +685,13 @@
   
   <xsd:simpleType name="LayerTransformEnum">
     <xsd:restriction base="xsd:token">
+       <xsd:enumeration value="fourbyteswap">
+         <xsd:annotation>
+           <xsd:documentation>
+             Swap bytes in 32 bit words.
+           </xsd:documentation>
+         </xsd:annotation>
+       </xsd:enumeration>
        <xsd:enumeration value="base64_MIME">
          <xsd:annotation>
            <xsd:documentation>
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/ByteSwapTransformer.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/ByteSwapTransformer.scala
new file mode 100644
index 0000000..858510c
--- /dev/null
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/ByteSwapTransformer.scala
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.layers
+
+import java.io.OutputStream
+import java.io.InputStream
+import java.util.ArrayDeque
+import java.util.Deque
+
+import org.apache.daffodil.exceptions.Assert
+import org.apache.daffodil.schema.annotation.props.gen.LayerLengthKind
+import org.apache.daffodil.schema.annotation.props.gen.LayerLengthUnits
+import org.apache.daffodil.util.Maybe
+import org.apache.daffodil.processors.TermRuntimeData
+import org.apache.daffodil.processors.LayerLengthInBytesEv
+import org.apache.daffodil.processors.LayerBoundaryMarkEv
+import org.apache.daffodil.processors.LayerCharsetEv
+import org.apache.daffodil.processors.parsers.PState
+import org.apache.daffodil.io.ExplicitLengthLimitingStream
+import org.apache.daffodil.processors.unparsers.UState
+
+/**
+ * An input stream wrapper that re-orders bytes according to wordsize.
+ *
+ * Bytes within the wrapped input stream are re-ordered wordsize bytes at a time.
+ * For example, if the wrapped input stream contains 10 bytes and wordsize is
+ * 4, then the bytes from the wrapped input stream are returned in the
+ * order 4 3 2 1 8 7 6 5 10 9.  If wordsize were 2 then the bytes from the
+ * wrapped input stream are returned in the order 2 1 4 3 6 5 8 7 10 9.
+ */
+class ByteSwapInputStream(wordsize: Int, jis: InputStream)
+  extends InputStream {
+
+  object State extends org.apache.daffodil.util.Enum {
+    abstract sealed trait Type extends EnumValueType
+
+    /**
+     * Buffering bytes in a word.
+     */
+    case object Filling extends Type
+
+    /**
+     * Return buffered bytes in a word in LIFO order.
+     */
+    case object Emptying extends Type
+
+    /**
+     * No more data.  Drain buffered bytes.
+     */
+    case object Draining extends Type
+
+    /**
+     *  No more data and no buffered bytes.
+     */
+    case object Done extends Type
+  }
+
+  private var c: Int = -2
+  private val stack: Deque[Int] = new ArrayDeque[Int](wordsize)
+  private var state: State.Type = State.Filling
+
+  /**
+   * Swap wordsize bytes at a time
+   *
+   */
+  override def read(): Int = {
+    import State._
+    if (state eq Done) return -1
+    while (state != Done) {
+      state match {
+        case Filling => {
+          c = jis.read()
+          if (c == -1) {
+            state = Draining
+          } else {
+            stack.push(c)
+            if (stack.size() == wordsize) {
+              state = Emptying
+            }
+          }
+        }
+        case Emptying => {
+          if (stack.isEmpty()) {
+            state = Filling
+          } else {
+            c = stack.pop()
+            return c
+          }
+        }
+        case Draining => {
+          if (stack.isEmpty()) {
+            state = Done
+            return -1
+          } else {
+            c = stack.pop()
+            return c
+          }
+        }
+        case Done =>
+          Assert.invariantFailed("Done state not allowed.")
+      }
+    }
+    Assert.invariantFailed("No fall through to here.")
+  }
+}
+
+/**
+ * An output stream wrapper that re-orders bytes according to wordsize.
+ *
+ * Bytes sent to the wrapped output stream are re-ordered wordsize bytes at a
+ * time.  For example, if 10 bytes are written and wordsize is 4, then the
+ * bytes are written to the wrapped output stream in the
+ * order 4 3 2 1 8 7 6 5 10 9.  If wordsize were 2 then the bytes are written
+ * to the wrapped output stream in the order 2 1 4 3 6 5 8 7 10 9.
+ */
+class ByteSwapOutputStream(wordsize: Int, jos: OutputStream)
+  extends OutputStream {
+
+  private val stack: Deque[Byte] = new ArrayDeque[Byte](wordsize)
+  private var closed = false
+
+  override def close(): Unit = {
+    if (!closed) {
+      while (!stack.isEmpty()) {
+        jos.write(stack.pop())
+      }
+      jos.close()
+      closed = true
+    }
+  }
+
+  override def write(bInt: Int): Unit = {
+    Assert.usage(!closed)
+    stack.push(bInt.toByte)
+    if (stack.size() == wordsize) {
+      while(!stack.isEmpty()) {
+        jos.write(stack.pop())
+      }
+    }
+  }
+}
+
+/**
+ * A LayerTransformer that re-orders bytes for the over/underlying layer.
+ */
+class ByteSwapTransformer(wordsize: Int, layerLengthInBytesEv: LayerLengthInBytesEv)
+  extends LayerTransformer() {
+
+  override def wrapLayerDecoder(jis: java.io.InputStream) = {
+    val s = new ByteSwapInputStream(wordsize, jis)
+    s
+  }
+
+  override def wrapLimitingStream(jis: java.io.InputStream, state: PState) = {
+    val layerLengthInBytes: Int = layerLengthInBytesEv.evaluate(state).toInt
+
+    val s = new ExplicitLengthLimitingStream(jis, layerLengthInBytes)
+    s
+  }
+
+  override protected def wrapLayerEncoder(jos: java.io.OutputStream): java.io.OutputStream = {
+    val s = new ByteSwapOutputStream(wordsize, jos)
+    s
+  }
+
+  override protected def wrapLimitingStream(jos: java.io.OutputStream, state: UState): java.io.OutputStream = {
+    jos // just return jos. The way the length will be used/stored is by way of
+    // taking the content length of the enclosing element. That will measure the
+    // length relative to the "ultimate" data output stream.
+  }
+}
+
+/**
+ * LayerTransformerFactory for ByteSwapTransformer instances.
+ *
+ * Requires that layerLengthKind="explicit".
+ */
+sealed abstract class ByteSwapTransformerFactory(wordsize: Int, name: String)
+  extends LayerTransformerFactory(name) {
+
+  override def newInstance(maybeLayerCharsetEv: Maybe[LayerCharsetEv],
+    maybeLayerLengthKind: Maybe[LayerLengthKind],
+    maybeLayerLengthInBytesEv: Maybe[LayerLengthInBytesEv],
+    maybeLayerLengthUnits: Maybe[LayerLengthUnits],
+    maybeLayerBoundaryMarkEv: Maybe[LayerBoundaryMarkEv],
+    trd: TermRuntimeData): LayerTransformer = {
+
+    trd.schemaDefinitionUnless(maybeLayerLengthKind.isDefined,
+      "The propert dfdl:layerLengthKind must be defined.")
+
+    val xformer =
+      maybeLayerLengthKind.get match {
+        case LayerLengthKind.Explicit => {
+          new ByteSwapTransformer(wordsize, maybeLayerLengthInBytesEv.get)
+        }
+        case x =>
+          trd.SDE("Property dfdl:layerLengthKind can only be 'explicit', but was '%s'",
+            x.toString)
+      }
+    xformer
+  }
+}
+
+/**
+ * Factory for ByteSwapTransformer instances with a wordsize of 4.
+ */
+object FourByteSwapTransformerFactory
+  extends ByteSwapTransformerFactory(4, "fourbyteswap")
+
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala
index 3687b5a..1e62410 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/layers/LayerTransformer.scala
@@ -101,6 +101,7 @@ object LayerTransformerFactory {
   register(IMFLineFoldedTransformerFactory)
   register(ICalendarLineFoldedTransformerFactory)
   register(AISPayloadArmoringTransformerFactory)
+  register(FourByteSwapTransformerFactory)
 }
 
 /**
diff --git a/daffodil-runtime1/src/test/scala/org/apache/daffodil/layers/TestByteSwapStream.scala b/daffodil-runtime1/src/test/scala/org/apache/daffodil/layers/TestByteSwapStream.scala
new file mode 100644
index 0000000..9fc279c
--- /dev/null
+++ b/daffodil-runtime1/src/test/scala/org/apache/daffodil/layers/TestByteSwapStream.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.layers
+
+import junit.framework.Assert._
+import java.io._
+import org.junit.Test
+import java.nio.charset.StandardCharsets
+
+class TestByteSwapStreams {
+
+  val iso8859 = StandardCharsets.ISO_8859_1
+
+  val unswapped32BitData = Array[Byte](0x76, 0x54, 0x32,        0x10,
+                                                   0xBA.toByte, 0x98.toByte)
+
+  val swapped32BitData = Array[Byte](0x10, 0x32, 0x54, 0x76, 0x98.toByte, 0xBA.toByte)
+
+  @Test def testFourByteSwapInputStream() = {
+    val data = unswapped32BitData
+    val bba = new ByteArrayInputStream(data)
+    val bss = new ByteSwapInputStream(4, bba)
+
+    val baos = new ByteArrayOutputStream()
+    var c: Int = -1
+    while ({
+      c = bss.read()
+      c != -1
+    }) {
+      baos.write(c)
+    }
+    baos.close()
+    val result = new String(baos.toByteArray(), iso8859)
+    val expected = new String(swapped32BitData, iso8859)
+    assertEquals(expected, result)
+  }
+
+  @Test def testFourByteSwapOutputStream() = {
+    val data = swapped32BitData
+    val bba = new ByteArrayInputStream(data)
+
+    val baos = new ByteArrayOutputStream()
+    val bsos = new ByteSwapOutputStream(4, baos)
+    var c: Int = -1
+    while ({
+      c = bba.read()
+      c != -1
+    }) {
+      bsos.write(c)
+    }
+    bsos.close()
+    baos.close()
+
+    val result = new String(baos.toByteArray(), iso8859)
+    val expected = new String(unswapped32BitData, iso8859)
+    assertEquals(expected, result)
+  }
+
+}