You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by sl...@apache.org on 2019/07/01 11:30:11 UTC
[incubator-daffodil] branch master updated: Adds hex/utf-8 data
dump on left over data
This is an automated email from the ASF dual-hosted git repository.
slawrence pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git
The following commit(s) were added to refs/heads/master by this push:
new f4c1a82 Adds hex/utf-8 data dump on left over data
f4c1a82 is described below
commit f4c1a822f26570adf2d6e7412c95e1795aff68cb
Author: Olabusayo Kilo <ok...@tresys.com>
AuthorDate: Thu Jun 27 15:09:12 2019 -0400
Adds hex/utf-8 data dump on left over data
- Fixes dumpTextLine to decode whole characters rather than per byte
in keeping with the updates to converToCharRepr
- Limited to 8 bytes displayed and decoded
- Uses Java utf-8 decoder
- 1 based indexing
DAFFODIL 1387
---
.../apache/daffodil/parsing/TestCLIParsing.scala | 10 ++++++---
.../src/main/scala/org/apache/daffodil/Main.scala | 16 +++++++++++++--
.../main/scala/org/apache/daffodil/io/Dump.scala | 15 +++++++-------
.../scala/org/apache/daffodil/io/TestDump.scala | 24 ++++++++++++++++++++++
4 files changed, 53 insertions(+), 12 deletions(-)
diff --git a/daffodil-cli/src/it/scala/org/apache/daffodil/parsing/TestCLIParsing.scala b/daffodil-cli/src/it/scala/org/apache/daffodil/parsing/TestCLIParsing.scala
index 9ba3a83..c9bbd7c 100644
--- a/daffodil-cli/src/it/scala/org/apache/daffodil/parsing/TestCLIParsing.scala
+++ b/daffodil-cli/src/it/scala/org/apache/daffodil/parsing/TestCLIParsing.scala
@@ -533,10 +533,10 @@ class TestCLIparsing {
val cmd = String.format("echo 0,1,2| %s parse -P parserThatDoesNotExist", Util.binPath, testSchemaFile)
shell.sendLine(cmd)
if (Util.isWindows) {
- shell.expect(contains("parserThatDoesNotExist (The system cannot find the file specified)"))
- } else {
+ shell.expect(contains("parserThatDoesNotExist (The system cannot find the file specified)"))
+ } else {
shell.expect(contains("parserThatDoesNotExist (No such file or directory)"))
- }
+ }
shell.sendLine("exit")
shell.expect(eof)
} finally {
@@ -816,6 +816,8 @@ class TestCLIparsing {
val cmd = String.format("echo 0,1,2,3,,,,| %s -t parse -s %s", Util.binPath, testSchemaFile)
shell.sendLine(cmd)
shell.expectIn(1, contains("Left over data. Consumed 56 bit(s) with at least"))
+ shell.expectIn(1, contains("Data (UTF-8) starting at byte 8 is: ("))
+ shell.expectIn(1, contains("Data (Hex) starting at byte 8 is: ("))
shell.sendLine("exit")
shell.expect(eof)
} finally {
@@ -833,6 +835,8 @@ class TestCLIparsing {
val cmd = String.format("echo 1,2,3,4,,,| %s parse -s %s -r matrix", Util.binPath, testSchemaFile)
shell.sendLine(cmd)
shell.expect(contains("Left over data. Consumed 56 bit(s) with at least"))
+ shell.expect(contains("Data (UTF-8) starting at byte 8 is: ("))
+ shell.expect(contains("Data (Hex) starting at byte 8 is: ("))
shell.sendLine("exit")
shell.expect(eof)
} finally {
diff --git a/daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala b/daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala
index 1d2e9b5..7d1f700 100644
--- a/daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala
+++ b/daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala
@@ -95,6 +95,8 @@ import org.apache.commons.io.IOUtils
import org.apache.daffodil.api.DaffodilTunables
import org.apache.daffodil.io.InputSourceDataInputStream
import org.apache.daffodil.tdml.TDMLTestNotCompatibleException
+import org.apache.daffodil.io.DataDumper
+import java.nio.ByteBuffer
class NullOutputStream extends OutputStream {
override def close() {}
@@ -905,13 +907,23 @@ object Main extends Logging {
}
} else {
// not streaming, show left over data warning
+ val Dump = new DataDumper
+ val curPositionInByte0b = inStream.inputSource.position
+ val bytesAvailable = inStream.inputSource.bytesAvailable
+ val bytesLimit = math.min(8, bytesAvailable).toInt
+ val destArray = new Array[Byte](bytesLimit)
+ val destArrayFilled = inStream.inputSource.get(destArray, 0, bytesLimit)
+ val dumpString = if (destArrayFilled) Dump.dump(Dump.TextOnly(Some("utf-8")), 0, destArray.length * 8, ByteBuffer.wrap(destArray), includeHeadingLine = false).mkString("\n") else ""
+ val dataText = if (destArrayFilled) s"\nData (UTF-8) starting at byte ${curPositionInByte0b + 1} is: (${dumpString}...)" else ""
+ val dataHex = if (destArrayFilled) s"\nData (Hex) starting at byte ${curPositionInByte0b + 1} is: (0x${destArray.map { a => f"$a%02x" }.mkString}...)" else ""
val remainingBits =
if (loc.bitLimit0b.isDefined) {
(loc.bitLimit0b.get - loc.bitPos0b).toString
} else {
- "at least " + (inStream.inputSource.bytesAvailable * 8)
+ "at least " + (bytesAvailable * 8)
}
- log(LogLevel.Warning, "Left over data. Consumed %s bit(s) with %s bit(s) remaining.", loc.bitPos0b, remainingBits)
+ val leftOverDataWarning = s"Left over data. Consumed ${loc.bitPos0b} bit(s) with ${remainingBits} bit(s) remaining." + dataText + dataHex
+ log(LogLevel.Warning, leftOverDataWarning)
keepParsing = false
error = true
}
diff --git a/daffodil-io/src/main/scala/org/apache/daffodil/io/Dump.scala b/daffodil-io/src/main/scala/org/apache/daffodil/io/Dump.scala
index 0e2c6eb..0d819b6 100644
--- a/daffodil-io/src/main/scala/org/apache/daffodil/io/Dump.scala
+++ b/daffodil-io/src/main/scala/org/apache/daffodil/io/Dump.scala
@@ -215,7 +215,7 @@ class DataDumper {
Assert.usage(lengthInBytes >= 0)
val (textDataHeader, textByteWidth, optEncName) = getTextParameters(optEncodingName)
- val decoder = getReplacingDecoder(optEncName)
+ val decoder = getReportingDecoder(optEncName)
val endByteAddress0b = math.max(startByteAddress0b + lengthInBytes - 1, 0)
val addressHeader = """87654321 """
@@ -438,14 +438,15 @@ class DataDumper {
}
}
- private def getReplacingDecoder(optEncodingName: Option[String]): Option[JavaCharsetDecoder] = {
+ private def getReportingDecoder(optEncodingName: Option[String]): Option[JavaCharsetDecoder] = {
val cs = optEncodingName.map { JavaCharset.forName(_) }
lazy val decoder = cs.map { _.newDecoder() }
decoder
}
/**
- * Decoder must be setup for REPLACE on decode error.
+ * Decoder must be setup for REPORT (default) on decode error.
+ * We will manually handle the replacing
*/
private def convertToCharRepr(
startingBytePos0b: Long,
@@ -598,13 +599,13 @@ class DataDumper {
val endByteAddress0b = math.max(startByteAddress0b + lengthInBytes - 1, 0)
// val cs = optEncodingName.map { Charset.forName(_) }
- val decoder = getReplacingDecoder(optEncodingName)
+ val decoder = getReportingDecoder(optEncodingName)
var i = startByteAddress0b
val sb = new StringBuilder
while (i <= endByteAddress0b) {
- val (cR, _, _) = convertToCharRepr(i - startByteAddress0b, endByteAddress0b, byteSource, decoder)
- sb += cR(0)
- i += 1
+ val (cR, nBytesConsumed, _) = convertToCharRepr(i - startByteAddress0b, endByteAddress0b, byteSource, decoder)
+ sb ++= cR
+ i += nBytesConsumed
}
val s = sb.mkString
val lines: Seq[String] = indicatorLine.toSeq :+ s
diff --git a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDump.scala b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDump.scala
index 86f3404..e97d51f 100644
--- a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDump.scala
+++ b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDump.scala
@@ -460,4 +460,28 @@ cø€␀␀␀wü␚’gU€␀gä 63f8 8000 0000 77fc 1a92 6755 8000 67e4 :00
assertEquals(expected, dumpString)
}
+ @Test def testDumpTextLine7() {
+ val data = """da8b f090 a487 f48b be8b be7a 1234
+ 4567 f48b 8018 0156 dada
+ 0000 0101 0817 dead beef cc7a"""
+ .replaceAll("\\s+", "").grouped(2)
+ .map { Integer.parseInt(_, 16).toByte }.toArray
+ val bs = new BS(data)
+ val lengthInbits = data.length * 8
+ val dumpString = Dump.dump(Dump.TextOnly(Some("utf-8")), 0, lengthInbits, bs).mkString("\n")
+ val uUnknown = 0xfffd
+ val arrayOfDecodedChars1 =
+ Array(0x068b, 0x10907, 0x10bf8b, uUnknown, 0x007a)
+ val arrayOfDecodedChars2 = Array(0x0034, 0x0045, 0x0067, uUnknown)
+ val arrayOfDecodedChars3 = Array(0x0056, uUnknown, uUnknown)
+ val arrayOfDecodedChars4 = Array(0x07ad, uUnknown, uUnknown, uUnknown, 0x007a)
+ val expected =
+ s"""${arrayOfDecodedChars1.map(Character.toChars(_).mkString).mkString}␒""" +
+ s"""${arrayOfDecodedChars2.map(Character.toChars(_).mkString).mkString}␘␁""" +
+ s"""${arrayOfDecodedChars3.map(Character.toChars(_).mkString).mkString}␀␀␁␁␈␗""" +
+ s"""${arrayOfDecodedChars4.map(Character.toChars(_).mkString).mkString}"""
+ .replace("\r\n", "\n")
+ assertEquals(expected, dumpString)
+ }
+
}