You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by ma...@apache.org on 2015/10/15 23:43:00 UTC
[1/6] lucy git commit: Customize Go binding for Token.
Repository: lucy
Updated Branches:
refs/heads/master 245ed3b36 -> 61d45291a
Customize Go binding for Token.
Hand-roll bindings for constructor, SetText, and GetText. Add basic
tests.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/7dd69a6b
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/7dd69a6b
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/7dd69a6b
Branch: refs/heads/master
Commit: 7dd69a6bbb82216da5dfb243c67c532af8c1c8df
Parents: 64156f4
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Tue Oct 6 13:41:25 2015 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Tue Oct 6 18:44:16 2015 -0700
----------------------------------------------------------------------
go/lucy/analysis.go | 50 +++++++++++++++++++++++++++++++++++++++++++
go/lucy/analysis_test.go | 19 ++++++++++++++++
2 files changed, 69 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/7dd69a6b/go/lucy/analysis.go
----------------------------------------------------------------------
diff --git a/go/lucy/analysis.go b/go/lucy/analysis.go
new file mode 100644
index 0000000..53cb3d9
--- /dev/null
+++ b/go/lucy/analysis.go
@@ -0,0 +1,50 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package lucy
+
+/*
+#include <stdlib.h>
+
+#include "Lucy/Analysis/Token.h"
+*/
+import "C"
+import "unsafe"
+
+import "git-wip-us.apache.org/repos/asf/lucy-clownfish.git/runtime/go/clownfish"
+
+func NewToken(text string) Token {
+ chars := C.CString(text)
+ defer C.free(unsafe.Pointer(chars))
+ size := C.size_t(len(text))
+ objC := C.lucy_Token_new(chars, size, 0, 0, 1.0, 1)
+ return WRAPToken(unsafe.Pointer(objC))
+}
+
+func (t *TokenIMP) SetText(text string) {
+ self := (*C.lucy_Token)(clownfish.Unwrap(t, "t"))
+ chars := C.CString(text)
+ defer C.free(unsafe.Pointer(chars))
+ size := C.size_t(len(text))
+ C.LUCY_Token_Set_Text(self, chars, size)
+}
+
+func (t *TokenIMP) GetText() string {
+ self := (*C.lucy_Token)(clownfish.Unwrap(t, "t"))
+ chars := C.LUCY_Token_Get_Text(self)
+ size := C.LUCY_Token_Get_Len(self)
+ return C.GoStringN(chars, C.int(size))
+}
http://git-wip-us.apache.org/repos/asf/lucy/blob/7dd69a6b/go/lucy/analysis_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/analysis_test.go b/go/lucy/analysis_test.go
index 26a30d5..2ae7a34 100644
--- a/go/lucy/analysis_test.go
+++ b/go/lucy/analysis_test.go
@@ -19,6 +19,25 @@ package lucy
import "testing"
import "reflect"
+func TestTokenBasics(t *testing.T) {
+ token := NewToken("foo")
+ if got := token.GetText(); got != "foo" {
+ t.Errorf("GetText: %v", got)
+ }
+ token.SetText("bar")
+ if got := token.GetText(); got != "bar" {
+ t.Errorf("SetText/GetText: %v", got)
+ }
+
+ // Verify that these were bound.
+ token.GetStartOffset()
+ token.GetEndOffset()
+ token.GetBoost()
+ token.GetPosInc()
+ token.GetPos()
+ token.GetLen()
+}
+
func TestRegexTokenizerSplit(t *testing.T) {
tokenizer := NewRegexTokenizer("\\S+")
var expected []interface{} = []interface{}{"foo", "bar", "baz"}
[4/6] lucy git commit: Test Go bindings for Inversion.
Posted by ma...@apache.org.
Test Go bindings for Inversion.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/4f7227e7
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/4f7227e7
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/4f7227e7
Branch: refs/heads/master
Commit: 4f7227e74b62e070dd1aae7e9c86b0cf388762dc
Parents: 7dd69a6
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Tue Oct 6 15:59:13 2015 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Tue Oct 6 18:44:17 2015 -0700
----------------------------------------------------------------------
go/build.go | 5 +++++
go/lucy/analysis_test.go | 25 +++++++++++++++++++++++++
2 files changed, 30 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/4f7227e7/go/build.go
----------------------------------------------------------------------
diff --git a/go/build.go b/go/build.go
index e250aeb..b265c2b 100644
--- a/go/build.go
+++ b/go/build.go
@@ -136,6 +136,11 @@ func runCFC() {
}
func specClasses(parcel *cfc.Parcel) {
+ tokenBinding := cfc.NewGoClass(parcel, "Lucy::Analysis::Token")
+ tokenBinding.SpecMethod("", "SetText(string)")
+ tokenBinding.SpecMethod("", "GetText() string")
+ tokenBinding.Register()
+
indexerBinding := cfc.NewGoClass(parcel, "Lucy::Index::Indexer")
indexerBinding.SpecMethod("", "Close() error")
indexerBinding.SpecMethod("Add_Doc", "AddDoc(doc interface{}) error")
http://git-wip-us.apache.org/repos/asf/lucy/blob/4f7227e7/go/lucy/analysis_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/analysis_test.go b/go/lucy/analysis_test.go
index 2ae7a34..300ae4c 100644
--- a/go/lucy/analysis_test.go
+++ b/go/lucy/analysis_test.go
@@ -38,6 +38,31 @@ func TestTokenBasics(t *testing.T) {
token.GetLen()
}
+func TestInversionBasics(t *testing.T) {
+ inv := NewInversion(nil)
+ inv.Append(NewToken("foo"))
+ inv.Append(NewToken("bar"))
+ inv.Append(NewToken("baz"))
+ if size := inv.GetSize(); size != 3 {
+ t.Errorf("Unexpected size: %d", size)
+ }
+
+ if got := inv.Next().GetText(); got != "foo" {
+ t.Errorf("Next yielded %s", got)
+ }
+
+ inv.Reset()
+ if got := inv.Next().GetText(); got != "foo" {
+ t.Errorf("Next after Reset yielded %s", got)
+ }
+
+ inv.Reset()
+ inv.Invert()
+ if got := inv.Next().GetText(); got != "bar" {
+ t.Errorf("Next after Invert yielded %s", got)
+ }
+}
+
func TestRegexTokenizerSplit(t *testing.T) {
tokenizer := NewRegexTokenizer("\\S+")
var expected []interface{} = []interface{}{"foo", "bar", "baz"}
[2/6] lucy git commit: Start new Go test file for Analysis.
Posted by ma...@apache.org.
Start new Go test file for Analysis.
Create test_analysis.go. Move a test.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/64156f43
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/64156f43
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/64156f43
Branch: refs/heads/master
Commit: 64156f43bc9b47c7e9cde954c34de5a9849a951b
Parents: a347c81
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Tue Oct 6 12:55:28 2015 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Tue Oct 6 18:44:16 2015 -0700
----------------------------------------------------------------------
go/lucy/analysis_test.go | 29 +++++++++++++++++++++++++++++
go/lucy/lucy_test.go | 10 ----------
2 files changed, 29 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/64156f43/go/lucy/analysis_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/analysis_test.go b/go/lucy/analysis_test.go
new file mode 100644
index 0000000..26a30d5
--- /dev/null
+++ b/go/lucy/analysis_test.go
@@ -0,0 +1,29 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package lucy
+
+import "testing"
+import "reflect"
+
+func TestRegexTokenizerSplit(t *testing.T) {
+ tokenizer := NewRegexTokenizer("\\S+")
+ var expected []interface{} = []interface{}{"foo", "bar", "baz"}
+ got := tokenizer.Split("foo bar baz")
+ if !reflect.DeepEqual(got, expected) {
+ t.Errorf("Expected %v, got %v", expected, got)
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucy/blob/64156f43/go/lucy/lucy_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/lucy_test.go b/go/lucy/lucy_test.go
index 8f11a3d..fce0435 100644
--- a/go/lucy/lucy_test.go
+++ b/go/lucy/lucy_test.go
@@ -18,7 +18,6 @@ package lucy
import "git-wip-us.apache.org/repos/asf/lucy-clownfish.git/runtime/go/clownfish"
import "testing"
-import "reflect"
type testDoc struct {
Content string
@@ -70,12 +69,3 @@ func TestOpenIndexer(t *testing.T) {
t.Error("Didn't catch exception opening indexer")
}
}
-
-func TestRegex(t *testing.T) {
- tokenizer := NewRegexTokenizer("\\S+")
- var expected []interface{} = []interface{}{"foo", "bar", "baz"}
- got := tokenizer.Split("foo bar baz")
- if !reflect.DeepEqual(got, expected) {
- t.Errorf("Expected %v, got %v", expected, got)
- }
-}
[3/6] lucy git commit: Customize Analyzer#Split for Go.
Posted by ma...@apache.org.
Customize Analyzer#Split for Go.
Return a slice of strings rather than a slice of empty interface.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/094b76eb
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/094b76eb
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/094b76eb
Branch: refs/heads/master
Commit: 094b76ebfd217278c40873cf4505344dc4c09e12
Parents: 4f7227e
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Tue Oct 6 18:41:58 2015 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Tue Oct 6 18:44:17 2015 -0700
----------------------------------------------------------------------
go/build.go | 4 ++++
go/lucy/analysis.go | 16 ++++++++++++++++
go/lucy/analysis_test.go | 2 +-
3 files changed, 21 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/094b76eb/go/build.go
----------------------------------------------------------------------
diff --git a/go/build.go b/go/build.go
index b265c2b..77263ee 100644
--- a/go/build.go
+++ b/go/build.go
@@ -141,6 +141,10 @@ func specClasses(parcel *cfc.Parcel) {
tokenBinding.SpecMethod("", "GetText() string")
tokenBinding.Register()
+ analyzerBinding := cfc.NewGoClass(parcel, "Lucy::Analysis::Analyzer")
+ analyzerBinding.SpecMethod("Split", "Split(string) []string")
+ analyzerBinding.Register()
+
indexerBinding := cfc.NewGoClass(parcel, "Lucy::Index::Indexer")
indexerBinding.SpecMethod("", "Close() error")
indexerBinding.SpecMethod("Add_Doc", "AddDoc(doc interface{}) error")
http://git-wip-us.apache.org/repos/asf/lucy/blob/094b76eb/go/lucy/analysis.go
----------------------------------------------------------------------
diff --git a/go/lucy/analysis.go b/go/lucy/analysis.go
index 53cb3d9..134c081 100644
--- a/go/lucy/analysis.go
+++ b/go/lucy/analysis.go
@@ -19,7 +19,9 @@ package lucy
/*
#include <stdlib.h>
+#include "Lucy/Analysis/Analyzer.h"
#include "Lucy/Analysis/Token.h"
+#include "Clownfish/Vector.h"
*/
import "C"
import "unsafe"
@@ -48,3 +50,17 @@ func (t *TokenIMP) GetText() string {
size := C.LUCY_Token_Get_Len(self)
return C.GoStringN(chars, C.int(size))
}
+
+func (a *AnalyzerIMP) Split(text string) []string {
+ self := (*C.lucy_Analyzer)(clownfish.Unwrap(a, "a"))
+ input := (*C.cfish_String)(clownfish.GoToClownfish(text, unsafe.Pointer(C.CFISH_STRING), false))
+ defer C.cfish_decref(unsafe.Pointer(input))
+ retvalCF := C.LUCY_Analyzer_Split(self, input)
+ size := C.CFISH_Vec_Get_Size(retvalCF)
+ retvalGO := make([]string, int(size))
+ for i := 0; i < int(size); i++ {
+ elem := C.CFISH_Vec_Fetch(retvalCF, C.size_t(i))
+ retvalGO[i] = clownfish.CFStringToGo(unsafe.Pointer(elem))
+ }
+ return retvalGO
+}
http://git-wip-us.apache.org/repos/asf/lucy/blob/094b76eb/go/lucy/analysis_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/analysis_test.go b/go/lucy/analysis_test.go
index 300ae4c..65a8403 100644
--- a/go/lucy/analysis_test.go
+++ b/go/lucy/analysis_test.go
@@ -65,7 +65,7 @@ func TestInversionBasics(t *testing.T) {
func TestRegexTokenizerSplit(t *testing.T) {
tokenizer := NewRegexTokenizer("\\S+")
- var expected []interface{} = []interface{}{"foo", "bar", "baz"}
+ expected := []string{"foo", "bar", "baz"}
got := tokenizer.Split("foo bar baz")
if !reflect.DeepEqual(got, expected) {
t.Errorf("Expected %v, got %v", expected, got)
[6/6] lucy git commit: Merge branch 'LUCY-282-test-analysis-go'
Posted by ma...@apache.org.
Merge branch 'LUCY-282-test-analysis-go'
Refine and test the Go bindings for Token, Inversion and various
Analyzers.
This closes #24.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/61d45291
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/61d45291
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/61d45291
Branch: refs/heads/master
Commit: 61d45291a5dffd85a26026b4ad7c9cd7adc14c3d
Parents: 245ed3b 51029f8
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Thu Oct 15 14:39:25 2015 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Thu Oct 15 14:39:25 2015 -0700
----------------------------------------------------------------------
go/build.go | 9 ++++
go/lucy/analysis.go | 66 ++++++++++++++++++++++++
go/lucy/analysis_test.go | 117 ++++++++++++++++++++++++++++++++++++++++++
go/lucy/lucy_test.go | 10 ----
4 files changed, 192 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/61d45291/go/build.go
----------------------------------------------------------------------
[5/6] lucy git commit: Test Go bindings for various Analyzers.
Posted by ma...@apache.org.
Test Go bindings for various Analyzers.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/51029f8f
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/51029f8f
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/51029f8f
Branch: refs/heads/master
Commit: 51029f8f6b5ad7c0d848e20db900cac5339c0bdb
Parents: 094b76e
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Tue Oct 6 18:42:58 2015 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Tue Oct 6 18:44:17 2015 -0700
----------------------------------------------------------------------
go/lucy/analysis_test.go | 44 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/51029f8f/go/lucy/analysis_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/analysis_test.go b/go/lucy/analysis_test.go
index 65a8403..5a068a6 100644
--- a/go/lucy/analysis_test.go
+++ b/go/lucy/analysis_test.go
@@ -63,6 +63,50 @@ func TestInversionBasics(t *testing.T) {
}
}
+func runAnalyzerTests(t *testing.T, a Analyzer) {
+ input := "foo bar baz"
+ fromSplit := a.Split(input)
+ fromTransform := make([]string, 0)
+ fromTransformText := make([]string, 0)
+ invFromTransform := a.Transform(NewInversion(NewToken(input)))
+ invFromTransformText := a.TransformText(input)
+ for i := 0; true; i++ {
+ token := invFromTransform.Next()
+ if token == nil {
+ break;
+ }
+ fromTransform = append(fromTransform, token.GetText())
+ }
+ for i := 0; true; i++ {
+ token := invFromTransformText.Next()
+ if token == nil {
+ break;
+ }
+ fromTransformText = append(fromTransformText, token.GetText())
+ }
+ if !reflect.DeepEqual(fromSplit, fromTransform) {
+ t.Errorf("Split and Transform not the same: %v, %v", fromSplit, fromTransform)
+ }
+ if !reflect.DeepEqual(fromSplit, fromTransformText) {
+ t.Errorf("Split and TransformText not the same: %v, %v", fromSplit, fromTransformText)
+ }
+
+ dupe := a.Load(a.Dump())
+ if !a.Equals(dupe) {
+ t.Errorf("Round-trip through Dump/Load failed: %v", dupe)
+ }
+}
+
+func TestCoreAnalyzers(t *testing.T) {
+ runAnalyzerTests(t, NewCaseFolder())
+ runAnalyzerTests(t, NewEasyAnalyzer("en"))
+ runAnalyzerTests(t, NewNormalizer("NFKC", true, false))
+ runAnalyzerTests(t, NewRegexTokenizer("\\S+"))
+ runAnalyzerTests(t, NewSnowballStemmer("en"))
+ runAnalyzerTests(t, NewSnowballStopFilter("en", nil))
+ runAnalyzerTests(t, NewStandardTokenizer())
+}
+
func TestRegexTokenizerSplit(t *testing.T) {
tokenizer := NewRegexTokenizer("\\S+")
expected := []string{"foo", "bar", "baz"}