You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by mb...@apache.org on 2019/07/16 16:18:28 UTC
[incubator-daffodil] 01/02: Adds support for non-spacing and
combining chars
This is an automated email from the ASF dual-hosted git repository.
mbeckerle pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git
commit 8555e25b14555b0ff4f271ea1170083a8d6c1a13
Author: Olabusayo Kilo <ok...@tresys.com>
AuthorDate: Tue Jul 9 18:04:27 2019 -0400
Adds support for non-spacing and combining chars
Pads non-spacing/combining chars with spaces so they show up properly
aligned in the debugger without combining with our ~ delimiters or
whitespace. Since they sometimes combine with preceding and following
characters, so we pad them on either side.
DAFFODIL-2168
---
.../main/scala/org/apache/daffodil/io/Dump.scala | 67 ++++++++++++++++++----
.../scala/org/apache/daffodil/io/TestDump.scala | 5 +-
2 files changed, 59 insertions(+), 13 deletions(-)
diff --git a/daffodil-io/src/main/scala/org/apache/daffodil/io/Dump.scala b/daffodil-io/src/main/scala/org/apache/daffodil/io/Dump.scala
index 0d819b6..a52541c 100644
--- a/daffodil-io/src/main/scala/org/apache/daffodil/io/Dump.scala
+++ b/daffodil-io/src/main/scala/org/apache/daffodil/io/Dump.scala
@@ -22,6 +22,7 @@ import java.nio.CharBuffer
import java.nio.charset.CodingErrorAction
import org.apache.daffodil.exceptions.Assert
import com.ibm.icu.lang.UCharacter
+import com.ibm.icu.lang.UCharacterEnums
import com.ibm.icu.lang.UProperty
import org.apache.daffodil.util.Misc
import org.apache.daffodil.equality._
@@ -419,6 +420,32 @@ class DataDumper {
}
/**
+ * Some characters act as combining marks and modify characters surrounding them.
+ * In order for us to display these characters as they are, we need to combine them with
+ * the appropriate number of spaces so they don't disturb other characters around them
+ */
+ private def homogenizeChars(codepoint: Int): (String, Int) = {
+ val charType = UCharacter.getType(codepoint)
+ val nCols = charNColumns(codepoint)
+ // supposed to always modify preceding character,
+ // but sometimes appears to modify succeeding character
+ // e.g \u093f or \u064d. Dual spacing is here to protect from this
+ charType match {
+ case UCharacterEnums.ECharacterCategory.COMBINING_SPACING_MARK => {
+ // can occupy spacing position by themselves
+ (" " + Character.toChars(codepoint).mkString + " ", nCols + 2)
+ }
+ case UCharacterEnums.ECharacterCategory.NON_SPACING_MARK => {
+ // do not occupy spacing position by themselves
+ // (so 1 space should be consumed, leaving an extra 1 for padding)
+ (" " + Character.toChars(codepoint).mkString + " ", nCols + 1)
+ }
+ case _ => {
+ ("" + Character.toChars(codepoint).mkString, nCols)
+ }
+ }
+ }
+ /**
* The width of the character in terms of how many "places" it uses up
* relative to a regular monospaced font character. This is for trying to get
* east asian and other double-wide characters to line up properly in columns.
@@ -473,6 +500,7 @@ class DataDumper {
bb.put(theByte)
}
bb.flip()
+
Assert.invariant(bb.remaining > 0)
var cr = CoderResult.OVERFLOW
var nConsumedBytes = 0
@@ -516,24 +544,41 @@ class DataDumper {
// Either way, we got our one character
Assert.invariant(nConsumedBytes > 0)
Assert.invariant(cb.hasArray)
- var allChars = cb.array
- remapped = if (allChars.length > 1) allChars.mkString else Misc.remapCodepointToVisibleGlyph(allChars(0)).toChar.toString
- nCols = if (allChars.length > 1) {
- try {
- var uCodePoint = UCharacter.getCodePoint(allChars(0), allChars(1))
- charNColumns(uCodePoint)
- } catch {
- case e: IllegalArgumentException => {
- allChars.mkString.length
+ val allChars = cb.array
+
+ val uCodePoint =
+ if (allChars.length > 1) {
+ try {
+ UCharacter.getCodePoint(allChars(0), allChars(1))
+ } catch {
+ case e: IllegalArgumentException => {
+ -1
+ }
+ }
+ } else allChars(0)
+
+ val (r: String, n: Int) =
+ if (allChars.length > 1) {
+ if (uCodePoint == -1) {
+ allChars.map(c => homogenizeChars(c)).foldLeft(("", 0)) {
+ (accForRemappedNcols, tupResultRemappedNcols) =>
+ (accForRemappedNcols._1 + tupResultRemappedNcols._1, //concat
+ accForRemappedNcols._2 + tupResultRemappedNcols._2) // add
+ }
+ } else {
+ homogenizeChars(uCodePoint)
}
+ } else {
+ homogenizeChars(Misc.remapCodepointToVisibleGlyph(allChars(0)))
}
- } else charNColumns(allChars(0))
+ remapped = r
+ nCols = n
}
(remapped, nConsumedBytes, nCols)
}
case None => {
// no encoding, so use the general one based on windows-1252 where
- // every byte corresponds ot a character with a glyph.
+ // every byte corresponds to a character with a glyph.
val byteValue = try {
bs.get(startingBytePos0b.toInt)
} catch {
diff --git a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDump.scala b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDump.scala
index e97d51f..d1dde1f 100644
--- a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDump.scala
+++ b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDump.scala
@@ -213,11 +213,12 @@ dada 0000 0101 0817 ece2 8017 ece2 dead beef cc7a 1234
val u10907 = Character.toChars(0x10907).mkString
val u10bf8b = Character.toChars(0x10bf8b).mkString
val u07ad = Character.toChars(0x07ad).mkString
+ val u0020 = Character.toChars(0x0020).mkString
val expected = s"""
87654321 0011 2233 4455 6677 8899 aabb ccdd eeff 0~1~2~3~4~5~6~7~8~9~a~b~c~d~e~f~
00000000: da8b f090 a487 f48b be8b be7a 1234 4567 ${u068b}~~~${u10907}~~~~~~~${u10bf8b}~~~~~~~�~z~␒~4~E~g~
00000010: f48b 8018 0156 dada 0000 0101 0817 ece2 �~~~~~␘~␁~V~�~�~␀~␀~␁~␁~␈~␗~�~�~
-00000020: 8017 ece2 dead beef cc7a 1234 4567 f48b ~~␗~�~�~${u07ad}~~~�~�~�~z~␒~4~E~g~�~~~
+00000020: 8017 ece2 dead beef cc7a 1234 4567 f48b ~~␗~�~�~${u0020}${u07ad}${u0020}~~�~�~�~z~␒~4~E~g~�~~~
""".replace("\r\n", "\n")
assertEquals(expected, "\n" + dumpString + "\n")
}
@@ -474,7 +475,7 @@ cø€␀␀␀wü␚’gU€␀gä 63f8 8000 0000 77fc 1a92 6755 8000 67e4 :00
Array(0x068b, 0x10907, 0x10bf8b, uUnknown, 0x007a)
val arrayOfDecodedChars2 = Array(0x0034, 0x0045, 0x0067, uUnknown)
val arrayOfDecodedChars3 = Array(0x0056, uUnknown, uUnknown)
- val arrayOfDecodedChars4 = Array(0x07ad, uUnknown, uUnknown, uUnknown, 0x007a)
+ val arrayOfDecodedChars4 = Array(0x0020, 0x07ad, 0x0020, uUnknown, uUnknown, uUnknown, 0x007a)
val expected =
s"""${arrayOfDecodedChars1.map(Character.toChars(_).mkString).mkString}␒""" +
s"""${arrayOfDecodedChars2.map(Character.toChars(_).mkString).mkString}␘␁""" +