You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by mb...@apache.org on 2019/07/16 16:18:28 UTC

[incubator-daffodil] 01/02: Adds support for non-spacing and combining chars

This is an automated email from the ASF dual-hosted git repository.

mbeckerle pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git

commit 8555e25b14555b0ff4f271ea1170083a8d6c1a13
Author: Olabusayo Kilo <ok...@tresys.com>
AuthorDate: Tue Jul 9 18:04:27 2019 -0400

    Adds support for non-spacing and combining chars
    
    Pads non-spacing/combining chars with spaces so they show up properly
    aligned in the debugger without combining with our ~ delimiters or
    whitespace. Since they sometimes combine with preceding and following
    characters, so we pad them on either side.
    
    DAFFODIL-2168
---
 .../main/scala/org/apache/daffodil/io/Dump.scala   | 67 ++++++++++++++++++----
 .../scala/org/apache/daffodil/io/TestDump.scala    |  5 +-
 2 files changed, 59 insertions(+), 13 deletions(-)

diff --git a/daffodil-io/src/main/scala/org/apache/daffodil/io/Dump.scala b/daffodil-io/src/main/scala/org/apache/daffodil/io/Dump.scala
index 0d819b6..a52541c 100644
--- a/daffodil-io/src/main/scala/org/apache/daffodil/io/Dump.scala
+++ b/daffodil-io/src/main/scala/org/apache/daffodil/io/Dump.scala
@@ -22,6 +22,7 @@ import java.nio.CharBuffer
 import java.nio.charset.CodingErrorAction
 import org.apache.daffodil.exceptions.Assert
 import com.ibm.icu.lang.UCharacter
+import com.ibm.icu.lang.UCharacterEnums
 import com.ibm.icu.lang.UProperty
 import org.apache.daffodil.util.Misc
 import org.apache.daffodil.equality._
@@ -419,6 +420,32 @@ class DataDumper {
   }
 
   /**
+   * Some characters act as combining marks and modify characters surrounding them.
+   * In order for us to display these characters as they are, we need to combine them with
+   * the appropriate number of spaces so they don't disturb other characters around them
+   */
+  private def homogenizeChars(codepoint: Int): (String, Int) = {
+    val charType = UCharacter.getType(codepoint)
+    val nCols = charNColumns(codepoint)
+    // supposed to always modify preceding character,
+    // but sometimes appears to modify succeeding character
+    // e.g \u093f or \u064d. Dual spacing is here to protect from this
+    charType match {
+      case UCharacterEnums.ECharacterCategory.COMBINING_SPACING_MARK => {
+        // can occupy spacing position by themselves
+        (" " + Character.toChars(codepoint).mkString + " ", nCols + 2)
+      }
+      case UCharacterEnums.ECharacterCategory.NON_SPACING_MARK => {
+        // do not occupy spacing position by themselves
+        // (so 1 space should be consumed, leaving an extra 1 for padding)
+        (" " + Character.toChars(codepoint).mkString + " ", nCols + 1)
+      }
+      case _ => {
+        ("" + Character.toChars(codepoint).mkString, nCols)
+      }
+    }
+  }
+  /**
    * The width of the character in terms of how many "places" it uses up
    * relative to a regular monospaced font character. This is for trying to get
    * east asian and other double-wide characters to line up properly in columns.
@@ -473,6 +500,7 @@ class DataDumper {
           bb.put(theByte)
         }
         bb.flip()
+
         Assert.invariant(bb.remaining > 0)
         var cr = CoderResult.OVERFLOW
         var nConsumedBytes = 0
@@ -516,24 +544,41 @@ class DataDumper {
           // Either way, we got our one character
           Assert.invariant(nConsumedBytes > 0)
           Assert.invariant(cb.hasArray)
-          var allChars = cb.array
-          remapped = if (allChars.length > 1) allChars.mkString else Misc.remapCodepointToVisibleGlyph(allChars(0)).toChar.toString
-          nCols = if (allChars.length > 1) {
-            try {
-              var uCodePoint = UCharacter.getCodePoint(allChars(0), allChars(1))
-              charNColumns(uCodePoint)
-            } catch {
-              case e: IllegalArgumentException => {
-                allChars.mkString.length
+          val allChars = cb.array
+
+          val uCodePoint =
+            if (allChars.length > 1) {
+              try {
+                UCharacter.getCodePoint(allChars(0), allChars(1))
+              } catch {
+                case e: IllegalArgumentException => {
+                  -1
+                }
+              }
+            } else allChars(0)
+
+          val (r: String, n: Int) =
+            if (allChars.length > 1) {
+              if (uCodePoint == -1) {
+                allChars.map(c => homogenizeChars(c)).foldLeft(("", 0)) {
+                  (accForRemappedNcols, tupResultRemappedNcols) =>
+                    (accForRemappedNcols._1 + tupResultRemappedNcols._1, //concat
+                      accForRemappedNcols._2 + tupResultRemappedNcols._2) // add
+                }
+              } else {
+                homogenizeChars(uCodePoint)
               }
+            } else {
+              homogenizeChars(Misc.remapCodepointToVisibleGlyph(allChars(0)))
             }
-          } else charNColumns(allChars(0))
+          remapped = r
+          nCols = n
         }
         (remapped, nConsumedBytes, nCols)
       }
       case None => {
         // no encoding, so use the general one based on windows-1252 where
-        // every byte corresponds ot a character with a glyph.
+        // every byte corresponds to a character with a glyph.
         val byteValue = try {
           bs.get(startingBytePos0b.toInt)
         } catch {
diff --git a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDump.scala b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDump.scala
index e97d51f..d1dde1f 100644
--- a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDump.scala
+++ b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestDump.scala
@@ -213,11 +213,12 @@ dada 0000 0101 0817 ece2 8017 ece2 dead beef cc7a 1234
     val u10907 = Character.toChars(0x10907).mkString
     val u10bf8b = Character.toChars(0x10bf8b).mkString
     val u07ad = Character.toChars(0x07ad).mkString
+    val u0020 = Character.toChars(0x0020).mkString
     val expected = s"""
 87654321  0011 2233 4455 6677 8899 aabb ccdd eeff  0~1~2~3~4~5~6~7~8~9~a~b~c~d~e~f~
 00000000: da8b f090 a487 f48b be8b be7a 1234 4567  ${u068b}~~~${u10907}~~~~~~~${u10bf8b}~~~~~~~�~z~␒~4~E~g~
 00000010: f48b 8018 0156 dada 0000 0101 0817 ece2  �~~~~~␘~␁~V~�~�~␀~␀~␁~␁~␈~␗~�~�~
-00000020: 8017 ece2 dead beef cc7a 1234 4567 f48b  ~~␗~�~�~${u07ad}~~~�~�~�~z~␒~4~E~g~�~~~
+00000020: 8017 ece2 dead beef cc7a 1234 4567 f48b  ~~␗~�~�~${u0020}${u07ad}${u0020}~~�~�~�~z~␒~4~E~g~�~~~
 """.replace("\r\n", "\n")
     assertEquals(expected, "\n" + dumpString + "\n")
   }
@@ -474,7 +475,7 @@ cø€␀␀␀wü␚’gU€␀gä  63f8 8000 0000 77fc 1a92 6755 8000 67e4 :00
       Array(0x068b, 0x10907, 0x10bf8b, uUnknown, 0x007a)
     val arrayOfDecodedChars2 = Array(0x0034, 0x0045, 0x0067, uUnknown)
     val arrayOfDecodedChars3 = Array(0x0056, uUnknown, uUnknown)
-    val arrayOfDecodedChars4 = Array(0x07ad, uUnknown, uUnknown, uUnknown, 0x007a)
+    val arrayOfDecodedChars4 = Array(0x0020, 0x07ad, 0x0020, uUnknown, uUnknown, uUnknown, 0x007a)
     val expected =
       s"""${arrayOfDecodedChars1.map(Character.toChars(_).mkString).mkString}␒""" +
         s"""${arrayOfDecodedChars2.map(Character.toChars(_).mkString).mkString}␘␁""" +