You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pdfbox.apache.org by ti...@apache.org on 2024/04/26 12:01:58 UTC
svn commit: r1917362 - in /pdfbox/branches/3.0/fontbox/src: main/java/org/apache/fontbox/ttf/gsub/CompoundCharacterTokenizer.java test/java/org/apache/fontbox/ttf/gsub/CompoundCharacterTokenizerTest.java
Author: tilman
Date: Fri Apr 26 12:01:58 2024
New Revision: 1917362
URL: http://svn.apache.org/viewvc?rev=1917362&view=rev
Log:
PDFBOX-5808: fix bug in CompoundCharacterTokenizer that prevented finding match after a match; add / remove tests and add validation by Fabrice Calafat
Modified:
pdfbox/branches/3.0/fontbox/src/main/java/org/apache/fontbox/ttf/gsub/CompoundCharacterTokenizer.java
pdfbox/branches/3.0/fontbox/src/test/java/org/apache/fontbox/ttf/gsub/CompoundCharacterTokenizerTest.java
Modified: pdfbox/branches/3.0/fontbox/src/main/java/org/apache/fontbox/ttf/gsub/CompoundCharacterTokenizer.java
URL: http://svn.apache.org/viewvc/pdfbox/branches/3.0/fontbox/src/main/java/org/apache/fontbox/ttf/gsub/CompoundCharacterTokenizer.java?rev=1917362&r1=1917361&r2=1917362&view=diff
==============================================================================
--- pdfbox/branches/3.0/fontbox/src/main/java/org/apache/fontbox/ttf/gsub/CompoundCharacterTokenizer.java (original)
+++ pdfbox/branches/3.0/fontbox/src/main/java/org/apache/fontbox/ttf/gsub/CompoundCharacterTokenizer.java Fri Apr 26 12:01:58 2024
@@ -33,36 +33,58 @@ import java.util.regex.Pattern;
*/
public class CompoundCharacterTokenizer
{
+ private static final String GLYPH_ID_SEPARATOR = "_";
private final Pattern regexExpression;
/**
* Constructor. Calls getRegexFromTokens which returns strings like
* (_79_99_)|(_80_99_)|(_92_99_) and creates a regexp assigned to regexExpression. See the code
* in GlyphArraySplitterRegexImpl on how these strings were created.
+ * <p>
+ * It is assumed the compound words are sorted in descending order of length.
*
* @param compoundWords A set of strings like _79_99_, _80_99_ or _92_99_ .
*/
public CompoundCharacterTokenizer(Set<String> compoundWords)
{
+ validateCompoundWords(compoundWords);
regexExpression = Pattern.compile(getRegexFromTokens(compoundWords));
}
- /**
- *
- * @param singleRegex
- * @deprecated use {@link CompoundCharacterTokenizer#CompoundCharacterTokenizer(Pattern)} instead.
- */
- @Deprecated
- public CompoundCharacterTokenizer(String singleRegex)
+ public CompoundCharacterTokenizer(Pattern pattern)
{
- regexExpression = Pattern.compile(singleRegex);
+ regexExpression = pattern;
}
- public CompoundCharacterTokenizer(Pattern pattern)
+ /**
+ * Validate the compound words. They should not be null or empty and should start and end with
+ * the GLYPH_ID_SEPARATOR
+ */
+ private void validateCompoundWords(Set<String> compoundWords)
{
- regexExpression = pattern;
+ if (compoundWords == null || compoundWords.isEmpty())
+ {
+ throw new IllegalArgumentException("Compound words cannot be null or empty");
+ }
+
+ // Ensure all word are starting and ending with the GLYPH_ID_SEPARATOR
+ compoundWords.forEach(word ->
+ {
+ if (!word.startsWith(GLYPH_ID_SEPARATOR) || !word.endsWith(GLYPH_ID_SEPARATOR))
+ {
+ throw new IllegalArgumentException(
+ "Compound words should start and end with " + GLYPH_ID_SEPARATOR);
+ }
+ });
}
+ /**
+ * Tokenize a string into tokens.
+ *
+ * @param text A string like "_66_71_71_74_79_70_"
+ * @return A list of tokens like "_66_", "_71_71_", "74_79_70_". The "_" is sometimes missing at
+ * the beginning or end, this has to be cleaned by the caller.
+ */
public List<String> tokenize(String text)
{
List<String> tokens = new ArrayList<>();
@@ -71,14 +93,14 @@ public class CompoundCharacterTokenizer
int lastIndexOfPrevMatch = 0;
- while (regexMatcher.find()) // this is where the magic happens:
+ while (regexMatcher.find(lastIndexOfPrevMatch)) // this is where the magic happens:
// the regexp is used to find a matching pattern for substitution
{
int beginIndexOfNextMatch = regexMatcher.start();
String prevToken = text.substring(lastIndexOfPrevMatch, beginIndexOfNextMatch);
- if (prevToken.length() > 0)
+ if (!prevToken.isEmpty())
{
tokens.add(prevToken);
}
@@ -88,12 +110,17 @@ public class CompoundCharacterTokenizer
tokens.add(currentMatch);
lastIndexOfPrevMatch = regexMatcher.end();
-
+ if (lastIndexOfPrevMatch < text.length() && text.charAt(lastIndexOfPrevMatch) != '_')
+ {
+ // beause it is sometimes positioned after the "_", but it should be positioned
+ // before the "_"
+ --lastIndexOfPrevMatch;
+ }
}
String tail = text.substring(lastIndexOfPrevMatch);
- if (tail.length() > 0)
+ if (!tail.isEmpty())
{
tokens.add(tail);
}
Modified: pdfbox/branches/3.0/fontbox/src/test/java/org/apache/fontbox/ttf/gsub/CompoundCharacterTokenizerTest.java
URL: http://svn.apache.org/viewvc/pdfbox/branches/3.0/fontbox/src/test/java/org/apache/fontbox/ttf/gsub/CompoundCharacterTokenizerTest.java?rev=1917362&r1=1917361&r2=1917362&view=diff
==============================================================================
--- pdfbox/branches/3.0/fontbox/src/test/java/org/apache/fontbox/ttf/gsub/CompoundCharacterTokenizerTest.java (original)
+++ pdfbox/branches/3.0/fontbox/src/test/java/org/apache/fontbox/ttf/gsub/CompoundCharacterTokenizerTest.java Fri Apr 26 12:01:58 2024
@@ -18,9 +18,9 @@
package org.apache.fontbox.ttf.gsub;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -28,143 +28,146 @@ import org.junit.jupiter.api.Test;
class CompoundCharacterTokenizerTest
{
-
@Test
- void testTokenize_happyPath_1()
+ void testTokenize_happyPath_2()
{
// given
CompoundCharacterTokenizer tokenizer = new CompoundCharacterTokenizer(
- new HashSet<>(Arrays.asList(new String[] { "HrkJj", "68RetP", "Yx!23uyt" })));
- String text = "12345HrkJjxabbcc68RetPxxxcfb1245678Yx!23uyt889000";
+ new HashSet<>(Arrays.asList(new String[] { "_84_93_", "_104_82_", "_104_87_" })));
+ String text = "_84_112_93_104_82_61_96_102_93_104_87_110_";
// when
List<String> tokens = tokenizer.tokenize(text);
// then
- assertEquals(text, String.join("", tokens));
- assertEquals(Arrays.asList("12345", "HrkJj", "xabbcc", "68RetP", "xxxcfb1245678",
- "Yx!23uyt", "889000"), tokens);
+ assertEquals(Arrays.asList("_84_112_93", "_104_82_", "_61_96_102_93", "_104_87_", "_110_"),
+ tokens);
}
@Test
- void testTokenize_happyPath_2()
+ void testTokenize_happyPath_3()
{
// given
CompoundCharacterTokenizer tokenizer = new CompoundCharacterTokenizer(
- new HashSet<>(Arrays.asList(new String[] { "84_93", "104_82", "104_87" })));
- String text = "84_112_93_104_82_61_96_102_93_104_87_110";
+ new HashSet<>(Arrays.asList(new String[] { "_67_112_96_", "_74_112_76_" })));
+ String text = "_67_112_96_103_93_108_93_";
// when
List<String> tokens = tokenizer.tokenize(text);
// then
- assertEquals(text, String.join("", tokens));
- assertEquals(Arrays.asList("84_112_93_", "104_82", "_61_96_102_93_", "104_87", "_110"),
- tokens);
+ assertEquals(Arrays.asList("_67_112_96_", "_103_93_108_93_"), tokens);
}
@Test
- void testTokenize_happyPath_3()
+ void testTokenize_happyPath_4()
{
// given
CompoundCharacterTokenizer tokenizer = new CompoundCharacterTokenizer(
- new HashSet<>(Arrays.asList(new String[] { "67_112_96", "74_112_76" })));
- String text = "67_112_96_103_93_108_93";
+ new HashSet<>(Arrays.asList(new String[] { "_67_112_96_", "_74_112_76_" })));
+ String text = "_94_67_112_96_112_91_103_";
// when
List<String> tokens = tokenizer.tokenize(text);
// then
- assertEquals(text, String.join("", tokens));
- assertEquals(Arrays.asList("67_112_96", "_103_93_108_93"), tokens);
+ assertEquals(Arrays.asList("_94", "_67_112_96_", "_112_91_103_"), tokens);
}
@Test
- void testTokenize_happyPath_4()
+ void testTokenize_happyPath_5()
{
// given
CompoundCharacterTokenizer tokenizer = new CompoundCharacterTokenizer(
- new HashSet<>(Arrays.asList(new String[] { "67_112_96", "74_112_76" })));
- String text = "94_67_112_96_112_91_103";
+ new HashSet<>(Arrays.asList(new String[] { "_67_112_", "_76_112_" })));
+ String text = "_94_167_112_91_103_";
// when
List<String> tokens = tokenizer.tokenize(text);
// then
- assertEquals(text, String.join("", tokens));
- assertEquals(Arrays.asList("94_", "67_112_96", "_112_91_103"), tokens);
+ assertEquals(Arrays.asList("_94_167_112_91_103_"), tokens);
+ }
+
+ @Test
+ void testTokenize_happyPath_6()
+ {
+
+ // given
+ CompoundCharacterTokenizer tokenizer = new CompoundCharacterTokenizer(
+ new HashSet<>(Arrays.asList("_100_", "_101_", "_102_", "_103_", "_104_")));
+ String text = "_100_101_102_103_104_";
+
+ // when
+ List<String> tokens = tokenizer.tokenize(text);
+
+ // then
+ assertEquals(Arrays.asList("_100_", "_101_", "_102_", "_103_", "_104_"), tokens);
}
@Test
- void testTokenize_happyPath_5()
+ void testTokenize_happyPath_7()
{
// given
CompoundCharacterTokenizer tokenizer = new CompoundCharacterTokenizer(
- new HashSet<>(Arrays.asList(new String[] { "67_112", "76_112" })));
- String text = "94_167_112_91_103";
+ new HashSet<>(Arrays.asList("_100_101_", "_102_", "_103_104_")));
+ String text = "_100_101_102_103_104_";
// when
List<String> tokens = tokenizer.tokenize(text);
// then
- assertEquals(text, String.join("", tokens));
- assertEquals(Arrays.asList("94_1", "67_112", "_91_103"), tokens);
+ assertEquals(Arrays.asList("_100_101_", "_102_", "_103_104_"), tokens);
}
@Test
- void testTokenize_regexAtStart()
+ void testTokenize_happyPath_8()
{
// given
CompoundCharacterTokenizer tokenizer = new CompoundCharacterTokenizer(
- new HashSet<>(Arrays.asList(new String[] { "HrkJj", "68RetP", "Yx!23uyt" })));
- String text = "Yx!23uyte12345HrkJjxabbcc68RetPxxxcfb1245678Yx!23uyt889000";
+ new HashSet<>(Arrays.asList("_100_101_102_", "_101_102_", "_103_104_")));
+ String text = "_100_101_102_103_104_";
// when
List<String> tokens = tokenizer.tokenize(text);
// then
- assertEquals(text, String.join("", tokens));
- assertEquals(0, tokens.indexOf("Yx!23uyt"));
+ assertEquals(Arrays.asList("_100_101_102_", "_103_104_"), tokens);
}
@Test
- void testTokenize_regexAtEnd()
+ void testTokenize_happyPath_9()
{
// given
CompoundCharacterTokenizer tokenizer = new CompoundCharacterTokenizer(
- new HashSet<>(Arrays.asList(new String[] { "HrkJj", "68RetP", "Yx!23uyt" })));
- String text = "Yx!23uyte12345HrkJjxabbcc68RetPxxxcfb1245678Yx!23uyt889000HrkJj";
+ new HashSet<>(Arrays.asList("_101_102_", "_101_102_")));
+ String text = "_100_101_102_103_104_";
// when
List<String> tokens = tokenizer.tokenize(text);
// then
- assertEquals(text, String.join("", tokens));
- assertEquals(0, tokens.indexOf("Yx!23uyt"));
- assertEquals(2, tokens.indexOf("HrkJj"));
- assertEquals(tokens.size() - 1, tokens.lastIndexOf("HrkJj"));
+ assertEquals(Arrays.asList("_100", "_101_102_", "_103_104_"), tokens);
}
@Test
- void testTokenize_Bangla()
+ void testTokenize_happyPath_10()
{
// given
- CompoundCharacterTokenizer tokenizer = new CompoundCharacterTokenizer(new HashSet<>(
- Arrays.asList(new String[] { "\u0995\u09cd\u09b7", "\u09aa\u09c1\u09a4\u09c1" })));
- String text = "\u0986\u09ae\u09bf \u0995\u09cb\u09a8 \u09aa\u09a5\u09c7 \u0995\u09cd\u09b7\u09c0\u09b0\u09c7\u09b0 \u09b7\u09a8\u09cd\u09a1 \u09aa\u09c1\u09a4\u09c1\u09b2 \u09b0\u09c1\u09aa\u09cb \u0997\u0999\u09cd\u0997\u09be \u098b\u09b7\u09bf";
+ CompoundCharacterTokenizer tokenizer = new CompoundCharacterTokenizer(
+ new HashSet<>(Arrays.asList("_201_", "_202_")));
+ String text = "_100_101_102_103_104_";
// when
List<String> tokens = tokenizer.tokenize(text);
// then
- assertEquals(text, String.join("", tokens));
- assertTrue(tokens.contains("\u0995\u09cd\u09b7"));
- assertTrue(tokens.contains("\u09aa\u09c1\u09a4\u09c1"));
+ assertEquals(Collections.singletonList("_100_101_102_103_104_"), tokens);
}
}