You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by nw...@apache.org on 2011/12/06 21:56:05 UTC
[lucy-commits] svn commit: r1211123 -
/incubator/lucy/branches/LUCY-196-uax-tokenizer/core/Lucy/Test/Analysis/TestStandardTokenizer.c
Author: nwellnhof
Date: Tue Dec 6 20:56:04 2011
New Revision: 1211123
URL: http://svn.apache.org/viewvc?rev=1211123&view=rev
Log:
Implement UCD word break tests
Modified:
incubator/lucy/branches/LUCY-196-uax-tokenizer/core/Lucy/Test/Analysis/TestStandardTokenizer.c
Modified: incubator/lucy/branches/LUCY-196-uax-tokenizer/core/Lucy/Test/Analysis/TestStandardTokenizer.c
URL: http://svn.apache.org/viewvc/incubator/lucy/branches/LUCY-196-uax-tokenizer/core/Lucy/Test/Analysis/TestStandardTokenizer.c?rev=1211123&r1=1211122&r2=1211123&view=diff
==============================================================================
--- incubator/lucy/branches/LUCY-196-uax-tokenizer/core/Lucy/Test/Analysis/TestStandardTokenizer.c (original)
+++ incubator/lucy/branches/LUCY-196-uax-tokenizer/core/Lucy/Test/Analysis/TestStandardTokenizer.c Tue Dec 6 20:56:04 2011
@@ -20,7 +20,8 @@
#include "Lucy/Test.h"
#include "Lucy/Test/Analysis/TestStandardTokenizer.h"
#include "Lucy/Analysis/StandardTokenizer.h"
-
+#include "Lucy/Store/FSFolder.h"
+#include "Lucy/Util/Json.h"
static void
test_Dump_Load_and_Equals(TestBatch *batch) {
@@ -40,6 +41,7 @@ test_Dump_Load_and_Equals(TestBatch *bat
static void
test_tokenizer(TestBatch *batch) {
StandardTokenizer *tokenizer = StandardTokenizer_new();
+
ZombieCharBuf *word = ZCB_WRAP_STR(
" ."
"tha\xCC\x82t's"
@@ -82,12 +84,40 @@ test_tokenizer(TestBatch *batch) {
&& CB_Equals_Str(token, "a", 1),
"Token: %s", CB_Get_Ptr8(token));
DECREF(got);
+
+ CharBuf *path = CB_newf("modules");
+ FSFolder *modules_folder = FSFolder_new(path);
+ if (!FSFolder_Check(modules_folder)) {
+ DECREF(modules_folder);
+ CB_setf(path, "../modules");
+ modules_folder = FSFolder_new(path);
+ if (!FSFolder_Check(modules_folder)) {
+ THROW(ERR, "Can't open modules folder");
+ }
+ }
+ CB_setf(path, "unicode/ucd/WordBreakTest.json");
+ VArray *tests = (VArray*)Json_slurp_json((Folder*)modules_folder, path);
+ if (!tests) { RETHROW(Err_get_error()); }
+
+ for (uint32_t i = 0, max = VA_Get_Size(tests); i < max; i++) {
+ Hash *test = (Hash*)VA_Fetch(tests, i);
+ CharBuf *text = (CharBuf*)Hash_Fetch_Str(test, "text", 4);
+ VArray *wanted = (VArray*)Hash_Fetch_Str(test, "words", 5);
+ VArray *got = StandardTokenizer_Split(tokenizer, text);
+ TEST_TRUE(batch, VA_Equals(wanted, (Obj*)got), "UCD test #%d", i + 1);
+ DECREF(got);
+ }
+
+ DECREF(tests);
+ DECREF(modules_folder);
+ DECREF(path);
+
DECREF(tokenizer);
}
void
TestStandardTokenizer_run_tests() {
- TestBatch *batch = TestBatch_new(6);
+ TestBatch *batch = TestBatch_new(984);
TestBatch_Plan(batch);