You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by nw...@apache.org on 2015/05/05 11:41:28 UTC
[1/9] lucy git commit: Dump unknown objects as strings
Repository: lucy
Updated Branches:
refs/heads/master 829c6d0a4 -> d460538f2
Dump unknown objects as strings
Fixes the previous commit that made Nums work.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/2aa3da22
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/2aa3da22
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/2aa3da22
Branch: refs/heads/master
Commit: 2aa3da221df320d00ef7d36d126f09b46013bae0
Parents: 829c6d0
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Tue May 5 11:13:14 2015 +0200
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Tue May 5 11:13:14 2015 +0200
----------------------------------------------------------------------
core/Lucy/Util/Freezer.c | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/2aa3da22/core/Lucy/Util/Freezer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/Freezer.c b/core/Lucy/Util/Freezer.c
index 7e053fd..91e6b9c 100644
--- a/core/Lucy/Util/Freezer.c
+++ b/core/Lucy/Util/Freezer.c
@@ -383,8 +383,11 @@ Freezer_dump(Obj *obj) {
else if (Obj_Is_A(obj, QUERY)) {
return Query_Dump((Query*)obj);
}
+ else if (Obj_Is_A(obj, NUM)) {
+ return Obj_Clone(obj);
+ }
else {
- return (Obj*)Obj_Clone(obj);
+ return (Obj*)Obj_To_String(obj);
}
}
[9/9] lucy git commit: Address memory leaks in Freezer test
Posted by nw...@apache.org.
Address memory leaks in Freezer test
Not sure what's the best way to fix thawing of BoolNums.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/d460538f
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/d460538f
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/d460538f
Branch: refs/heads/master
Commit: d460538f26959f7bce097f5e9a11aa4616e2671f
Parents: 4e18b58
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Tue May 5 11:34:17 2015 +0200
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Tue May 5 11:34:17 2015 +0200
----------------------------------------------------------------------
core/Lucy/Test/Util/TestFreezer.c | 1 +
core/Lucy/Util/Freezer.c | 2 ++
2 files changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/d460538f/core/Lucy/Test/Util/TestFreezer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Test/Util/TestFreezer.c b/core/Lucy/Test/Util/TestFreezer.c
index 7657a00..01fb687 100644
--- a/core/Lucy/Test/Util/TestFreezer.c
+++ b/core/Lucy/Test/Util/TestFreezer.c
@@ -95,6 +95,7 @@ test_hash(TestBatchRunner *runner) {
String *str = TestUtils_random_string(rand() % 1200);
Integer32 *num = Int32_new(i);
Hash_Store(wanted, str, (Obj*)num);
+ DECREF(str);
}
{
http://git-wip-us.apache.org/repos/asf/lucy/blob/d460538f/core/Lucy/Util/Freezer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/Freezer.c b/core/Lucy/Util/Freezer.c
index 621d63c..9cc9410 100644
--- a/core/Lucy/Util/Freezer.c
+++ b/core/Lucy/Util/Freezer.c
@@ -142,6 +142,8 @@ Freezer_deserialize(Obj *obj, InStream *instream) {
if (Obj_Is_A(obj, BOOLNUM)) {
bool value = !!InStream_Read_U8(instream);
Obj *result = value ? INCREF(CFISH_TRUE) : INCREF(CFISH_FALSE);
+ // FIXME: This DECREF is essentially a no-op causing a
+ // memory leak.
DECREF(obj);
obj = result;
}
[7/9] lucy git commit: Eliminate ViewBB in STORABLE_thaw
Posted by nw...@apache.org.
Eliminate ViewBB in STORABLE_thaw
This results in an additional copy of the buffer. To avoid the copy,
we'd need something like a read-only RAMFile backed by a Blob. But this
code path isn't performance-critical.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/c6ccfdb6
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/c6ccfdb6
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/c6ccfdb6
Branch: refs/heads/master
Commit: c6ccfdb6be608d2c37f257f7ad0cbf7c83582449
Parents: 5651dd3
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Mon May 4 18:36:52 2015 +0200
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Tue May 5 11:23:38 2015 +0200
----------------------------------------------------------------------
perl/buildlib/Lucy/Build/Binding/Misc.pm | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/c6ccfdb6/perl/buildlib/Lucy/Build/Binding/Misc.pm
----------------------------------------------------------------------
diff --git a/perl/buildlib/Lucy/Build/Binding/Misc.pm b/perl/buildlib/Lucy/Build/Binding/Misc.pm
index dbc54d0..8d20418 100644
--- a/perl/buildlib/Lucy/Build/Binding/Misc.pm
+++ b/perl/buildlib/Lucy/Build/Binding/Misc.pm
@@ -119,8 +119,8 @@ PPCODE:
= cfish_Class_singleton((cfish_String*)class_name_str, NULL);
STRLEN len;
char *ptr = SvPV(serialized_sv, len);
- cfish_ViewByteBuf *contents = cfish_ViewBB_new(ptr, len);
- lucy_RAMFile *ram_file = lucy_RAMFile_new((cfish_ByteBuf*)contents, true);
+ cfish_ByteBuf *contents = cfish_BB_new_bytes(ptr, len);
+ lucy_RAMFile *ram_file = lucy_RAMFile_new(contents, true);
lucy_RAMFileHandle *file_handle
= lucy_RAMFH_open(NULL, LUCY_FH_READ_ONLY, ram_file);
lucy_InStream *instream = lucy_InStream_open((cfish_Obj*)file_handle);
[3/9] lucy git commit: Switch Freezer and field_buf over to Blob
Posted by nw...@apache.org.
Switch Freezer and field_buf over to Blob
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/9b2f5678
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/9b2f5678
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/9b2f5678
Branch: refs/heads/master
Commit: 9b2f56786319685eb51132002773e2600e6b1d5a
Parents: 4313e9a
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Mon May 4 16:25:51 2015 +0200
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Tue May 5 11:23:05 2015 +0200
----------------------------------------------------------------------
core/Lucy/Index/DocVector.c | 15 ++++++++-------
core/Lucy/Index/DocVector.cfh | 4 ++--
core/Lucy/Index/HighlightReader.c | 3 ++-
core/Lucy/Index/HighlightWriter.c | 11 +++++++----
core/Lucy/Index/HighlightWriter.cfh | 2 +-
core/Lucy/Test/Util/TestFreezer.c | 11 ++++++-----
core/Lucy/Util/Freezer.c | 29 +++++++++++++++--------------
core/Lucy/Util/Freezer.cfh | 10 +++++-----
8 files changed, 46 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/9b2f5678/core/Lucy/Index/DocVector.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/DocVector.c b/core/Lucy/Index/DocVector.c
index de393c8..b9619b7 100644
--- a/core/Lucy/Index/DocVector.c
+++ b/core/Lucy/Index/DocVector.c
@@ -19,6 +19,7 @@
#include "Lucy/Index/DocVector.h"
+#include "Clownfish/Blob.h"
#include "Clownfish/CharBuf.h"
#include "Lucy/Index/TermVector.h"
#include "Lucy/Store/InStream.h"
@@ -28,7 +29,7 @@
// Extract a document's compressed TermVector data into (term_text =>
// compressed positional data) pairs.
static Hash*
-S_extract_tv_cache(ByteBuf *field_buf);
+S_extract_tv_cache(Blob *field_buf);
// Pull a TermVector object out from compressed positional data.
static TermVector*
@@ -74,15 +75,15 @@ DocVec_Destroy_IMP(DocVector *self) {
void
DocVec_Add_Field_Buf_IMP(DocVector *self, String *field,
- ByteBuf *field_buf) {
+ Blob *field_buf) {
DocVectorIVARS *const ivars = DocVec_IVARS(self);
Hash_Store(ivars->field_bufs, field, INCREF(field_buf));
}
-ByteBuf*
+Blob*
DocVec_Field_Buf_IMP(DocVector *self, String *field) {
DocVectorIVARS *const ivars = DocVec_IVARS(self);
- return (ByteBuf*)Hash_Fetch(ivars->field_bufs, field);
+ return (Blob*)Hash_Fetch(ivars->field_bufs, field);
}
Vector*
@@ -99,7 +100,7 @@ DocVec_Term_Vector_IMP(DocVector *self, String *field,
// If no cache hit, try to fill cache.
if (field_vector == NULL) {
- ByteBuf *field_buf = (ByteBuf*)Hash_Fetch(ivars->field_bufs, field);
+ Blob *field_buf = (Blob*)Hash_Fetch(ivars->field_bufs, field);
// Bail if there's no content or the field isn't highlightable.
if (field_buf == NULL) { return NULL; }
@@ -118,9 +119,9 @@ DocVec_Term_Vector_IMP(DocVector *self, String *field,
}
static Hash*
-S_extract_tv_cache(ByteBuf *field_buf) {
+S_extract_tv_cache(Blob *field_buf) {
Hash *tv_cache = Hash_new(0);
- const char *tv_string = BB_Get_Buf(field_buf);
+ const char *tv_string = Blob_Get_Buf(field_buf);
int32_t num_terms = NumUtil_decode_c32(&tv_string);
CharBuf *text_buf = CB_new(0);
http://git-wip-us.apache.org/repos/asf/lucy/blob/9b2f5678/core/Lucy/Index/DocVector.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/DocVector.cfh b/core/Lucy/Index/DocVector.cfh
index 0dde0a1..8a801ad 100644
--- a/core/Lucy/Index/DocVector.cfh
+++ b/core/Lucy/Index/DocVector.cfh
@@ -39,12 +39,12 @@ class Lucy::Index::DocVector nickname DocVec
/** Add a compressed, encoded TermVector to the object.
*/
void
- Add_Field_Buf(DocVector *self, String *field, ByteBuf *field_buf);
+ Add_Field_Buf(DocVector *self, String *field, Blob *field_buf);
/** Return the compressed, encoded TermVector associated with a particular
* field.
*/
- ByteBuf*
+ Blob*
Field_Buf(DocVector *self, String *field);
public void
http://git-wip-us.apache.org/repos/asf/lucy/blob/9b2f5678/core/Lucy/Index/HighlightReader.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/HighlightReader.c b/core/Lucy/Index/HighlightReader.c
index 345431e..eff0e32 100644
--- a/core/Lucy/Index/HighlightReader.c
+++ b/core/Lucy/Index/HighlightReader.c
@@ -19,6 +19,7 @@
#define C_LUCY_DEFAULTHIGHLIGHTREADER
#include "Lucy/Util/ToolSet.h"
+#include "Clownfish/Blob.h"
#include "Lucy/Index/HighlightReader.h"
#include "Lucy/Index/DocVector.h"
#include "Lucy/Index/HighlightWriter.h"
@@ -201,7 +202,7 @@ DefHLReader_Fetch_Doc_Vec_IMP(DefaultHighlightReader *self, int32_t doc_id) {
uint32_t num_fields = InStream_Read_C32(dat_in);
while (num_fields--) {
String *field = Freezer_read_string(dat_in);
- ByteBuf *field_buf = Freezer_read_bytebuf(dat_in);
+ Blob *field_buf = Freezer_read_blob(dat_in);
DocVec_Add_Field_Buf(doc_vec, field, field_buf);
DECREF(field_buf);
DECREF(field);
http://git-wip-us.apache.org/repos/asf/lucy/blob/9b2f5678/core/Lucy/Index/HighlightWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/HighlightWriter.c b/core/Lucy/Index/HighlightWriter.c
index 9655455..fbb4e91 100644
--- a/core/Lucy/Index/HighlightWriter.c
+++ b/core/Lucy/Index/HighlightWriter.c
@@ -20,6 +20,7 @@
#include <stdio.h>
+#include "Clownfish/Blob.h"
#include "Lucy/Index/HighlightWriter.h"
#include "Lucy/Analysis/Token.h"
#include "Lucy/Analysis/Inversion.h"
@@ -129,15 +130,15 @@ HLWriter_Add_Inverted_Doc_IMP(HighlightWriter *self, Inverter *inverter,
) {
String *field = Inverter_Get_Field_Name(inverter);
Inversion *inversion = Inverter_Get_Inversion(inverter);
- ByteBuf *tv_buf = HLWriter_TV_Buf(self, inversion);
+ Blob *tv_buf = HLWriter_TV_Buf(self, inversion);
Freezer_serialize_string(field, dat_out);
- Freezer_serialize_bytebuf(tv_buf, dat_out);
+ Freezer_serialize_blob(tv_buf, dat_out);
DECREF(tv_buf);
}
}
}
-ByteBuf*
+Blob*
HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) {
const char *last_text = "";
size_t last_len = 0;
@@ -204,7 +205,9 @@ HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) {
char *dest = BB_Get_Buf(tv_buf);
NumUtil_encode_padded_c32(num_postings, &dest);
- return tv_buf;
+ Blob *blob = BB_Yield_Blob(tv_buf);
+ DECREF(tv_buf);
+ return blob;
}
void
http://git-wip-us.apache.org/repos/asf/lucy/blob/9b2f5678/core/Lucy/Index/HighlightWriter.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/HighlightWriter.cfh b/core/Lucy/Index/HighlightWriter.cfh
index 202509d..00619aa 100644
--- a/core/Lucy/Index/HighlightWriter.cfh
+++ b/core/Lucy/Index/HighlightWriter.cfh
@@ -35,7 +35,7 @@ class Lucy::Index::HighlightWriter nickname HLWriter
init(HighlightWriter *self, Schema *schema, Snapshot *snapshot,
Segment *segment, PolyReader *polyreader);
- incremented ByteBuf*
+ incremented Blob*
TV_Buf(HighlightWriter *self, Inversion *inversion);
public void
http://git-wip-us.apache.org/repos/asf/lucy/blob/9b2f5678/core/Lucy/Test/Util/TestFreezer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Test/Util/TestFreezer.c b/core/Lucy/Test/Util/TestFreezer.c
index 2b924b9..7657a00 100644
--- a/core/Lucy/Test/Util/TestFreezer.c
+++ b/core/Lucy/Test/Util/TestFreezer.c
@@ -18,6 +18,7 @@
#define TESTLUCY_USE_SHORT_NAMES
#include "Lucy/Util/ToolSet.h"
+#include "Clownfish/Blob.h"
#include "Clownfish/TestHarness/TestBatchRunner.h"
#include "Clownfish/TestHarness/TestUtils.h"
#include "Lucy/Test/Util/TestFreezer.h"
@@ -67,10 +68,10 @@ S_dump_load(Obj *object) {
}
static void
-test_bytebuf(TestBatchRunner *runner) {
- ByteBuf *wanted = BB_new_bytes("foobar", 6);
- ByteBuf *got = (ByteBuf*)S_freeze_thaw((Obj*)wanted);
- TEST_TRUE(runner, got && BB_Equals(wanted, (Obj*)got),
+test_blob(TestBatchRunner *runner) {
+ Blob *wanted = Blob_new("foobar", 6);
+ Blob *got = (Blob*)S_freeze_thaw((Obj*)wanted);
+ TEST_TRUE(runner, got && Blob_Equals(wanted, (Obj*)got),
"Serialization round trip");
DECREF(wanted);
DECREF(got);
@@ -172,7 +173,7 @@ test_varray(TestBatchRunner *runner) {
void
TestFreezer_Run_IMP(TestFreezer *self, TestBatchRunner *runner) {
TestBatchRunner_Plan(runner, (TestBatch*)self, 11);
- test_bytebuf(runner);
+ test_blob(runner);
test_string(runner);
test_hash(runner);
test_num(runner);
http://git-wip-us.apache.org/repos/asf/lucy/blob/9b2f5678/core/Lucy/Util/Freezer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/Freezer.c b/core/Lucy/Util/Freezer.c
index 91e6b9c..621d63c 100644
--- a/core/Lucy/Util/Freezer.c
+++ b/core/Lucy/Util/Freezer.c
@@ -17,6 +17,7 @@
#define C_LUCY_FREEZER
#include "Lucy/Util/ToolSet.h"
+#include "Clownfish/Blob.h"
#include "Clownfish/HashIterator.h"
#include "Lucy/Util/Freezer.h"
#include "Lucy/Store/InStream.h"
@@ -54,8 +55,8 @@ Freezer_serialize(Obj *obj, OutStream *outstream) {
if (Obj_Is_A(obj, STRING)) {
Freezer_serialize_string((String*)obj, outstream);
}
- else if (Obj_Is_A(obj, BYTEBUF)) {
- Freezer_serialize_bytebuf((ByteBuf*)obj, outstream);
+ else if (Obj_Is_A(obj, BLOB)) {
+ Freezer_serialize_blob((Blob*)obj, outstream);
}
else if (Obj_Is_A(obj, VECTOR)) {
Freezer_serialize_varray((Vector*)obj, outstream);
@@ -127,8 +128,8 @@ Freezer_deserialize(Obj *obj, InStream *instream) {
if (Obj_Is_A(obj, STRING)) {
obj = (Obj*)Freezer_deserialize_string((String*)obj, instream);
}
- else if (Obj_Is_A(obj, BYTEBUF)) {
- obj = (Obj*)Freezer_deserialize_bytebuf((ByteBuf*)obj, instream);
+ else if (Obj_Is_A(obj, BLOB)) {
+ obj = (Obj*)Freezer_deserialize_blob((Blob*)obj, instream);
}
else if (Obj_Is_A(obj, VECTOR)) {
obj = (Obj*)Freezer_deserialize_varray((Vector*)obj, instream);
@@ -229,24 +230,24 @@ Freezer_read_string(InStream *instream) {
}
void
-Freezer_serialize_bytebuf(ByteBuf *bytebuf, OutStream *outstream) {
- size_t size = BB_Get_Size(bytebuf);
+Freezer_serialize_blob(Blob *blob, OutStream *outstream) {
+ size_t size = Blob_Get_Size(blob);
OutStream_Write_C32(outstream, size);
- OutStream_Write_Bytes(outstream, BB_Get_Buf(bytebuf), size);
+ OutStream_Write_Bytes(outstream, Blob_Get_Buf(blob), size);
}
-ByteBuf*
-Freezer_deserialize_bytebuf(ByteBuf *bytebuf, InStream *instream) {
+Blob*
+Freezer_deserialize_blob(Blob *blob, InStream *instream) {
size_t size = InStream_Read_C32(instream);
char *buf = (char*)MALLOCATE(size);
InStream_Read_Bytes(instream, buf, size);
- return BB_init_steal_bytes(bytebuf, buf, size, size);
+ return Blob_init_steal(blob, buf, size);
}
-ByteBuf*
-Freezer_read_bytebuf(InStream *instream) {
- ByteBuf *bytebuf = (ByteBuf*)Class_Make_Obj(BYTEBUF);
- return Freezer_deserialize_bytebuf(bytebuf, instream);
+Blob*
+Freezer_read_blob(InStream *instream) {
+ Blob *blob = (Blob*)Class_Make_Obj(BLOB);
+ return Freezer_deserialize_blob(blob, instream);
}
void
http://git-wip-us.apache.org/repos/asf/lucy/blob/9b2f5678/core/Lucy/Util/Freezer.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/Freezer.cfh b/core/Lucy/Util/Freezer.cfh
index ef6da38..71af3ce 100644
--- a/core/Lucy/Util/Freezer.cfh
+++ b/core/Lucy/Util/Freezer.cfh
@@ -65,13 +65,13 @@ inert class Lucy::Util::Freezer {
read_string(InStream *instream);
inert void
- serialize_bytebuf(ByteBuf *bytebuf, OutStream *outstream);
+ serialize_blob(Blob *blob, OutStream *outstream);
- inert incremented ByteBuf*
- deserialize_bytebuf(decremented ByteBuf *bytebuf, InStream *instream);
+ inert incremented Blob*
+ deserialize_blob(decremented Blob *blob, InStream *instream);
- inert incremented ByteBuf*
- read_bytebuf(InStream *instream);
+ inert incremented Blob*
+ read_blob(InStream *instream);
inert void
serialize_varray(Vector *array, OutStream *outstream);
[5/9] lucy git commit: Switch MockMatcher over to Blob
Posted by nw...@apache.org.
Switch MockMatcher over to Blob
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/5651dd32
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/5651dd32
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/5651dd32
Branch: refs/heads/master
Commit: 5651dd329a8e084beede06305712ad43db712d9b
Parents: 2f7df54
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Mon May 4 18:04:19 2015 +0200
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Tue May 5 11:23:05 2015 +0200
----------------------------------------------------------------------
core/LucyX/Search/MockMatcher.c | 9 +++++----
core/LucyX/Search/MockMatcher.cfh | 6 +++---
perl/lib/LucyX/Search/MockMatcher.pm | 2 +-
3 files changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/5651dd32/core/LucyX/Search/MockMatcher.c
----------------------------------------------------------------------
diff --git a/core/LucyX/Search/MockMatcher.c b/core/LucyX/Search/MockMatcher.c
index 867d7a2..73a1b8d 100644
--- a/core/LucyX/Search/MockMatcher.c
+++ b/core/LucyX/Search/MockMatcher.c
@@ -18,21 +18,22 @@
#include "Lucy/Util/ToolSet.h"
#include "LucyX/Search/MockMatcher.h"
+#include "Clownfish/Blob.h"
MockMatcher*
-MockMatcher_new(I32Array *doc_ids, ByteBuf *scores) {
+MockMatcher_new(I32Array *doc_ids, Blob *scores) {
MockMatcher *self = (MockMatcher*)Class_Make_Obj(MOCKMATCHER);
return MockMatcher_init(self, doc_ids, scores);
}
MockMatcher*
-MockMatcher_init(MockMatcher *self, I32Array *doc_ids, ByteBuf *scores) {
+MockMatcher_init(MockMatcher *self, I32Array *doc_ids, Blob *scores) {
Matcher_init((Matcher*)self);
MockMatcherIVARS *const ivars = MockMatcher_IVARS(self);
ivars->tick = -1;
ivars->size = I32Arr_Get_Size(doc_ids);
ivars->doc_ids = (I32Array*)INCREF(doc_ids);
- ivars->scores = (ByteBuf*)INCREF(scores);
+ ivars->scores = (Blob*)INCREF(scores);
return self;
}
@@ -60,7 +61,7 @@ MockMatcher_Score_IMP(MockMatcher* self) {
if (!ivars->scores) {
THROW(ERR, "Can't call Score() unless scores supplied");
}
- const float *raw_scores = (const float*)BB_Get_Buf(ivars->scores);
+ const float *raw_scores = (const float*)Blob_Get_Buf(ivars->scores);
return raw_scores[ivars->tick];
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/5651dd32/core/LucyX/Search/MockMatcher.cfh
----------------------------------------------------------------------
diff --git a/core/LucyX/Search/MockMatcher.cfh b/core/LucyX/Search/MockMatcher.cfh
index 4115575..88666aa 100644
--- a/core/LucyX/Search/MockMatcher.cfh
+++ b/core/LucyX/Search/MockMatcher.cfh
@@ -20,11 +20,11 @@ class LucyX::Search::MockMatcher inherits Lucy::Search::Matcher {
size_t size;
I32Array *doc_ids;
- ByteBuf *scores;
+ Blob *scores;
int32_t tick;
inert incremented MockMatcher*
- new(I32Array *doc_ids, ByteBuf *scores = NULL);
+ new(I32Array *doc_ids, Blob *scores = NULL);
/**
* @param doc_ids An array of matching doc ids.
@@ -32,7 +32,7 @@ class LucyX::Search::MockMatcher inherits Lucy::Search::Matcher {
* supplied, calling [](cfish:.Score) will throw an exception.
*/
inert incremented MockMatcher*
- init(MockMatcher *self, I32Array *doc_ids, ByteBuf *scores = NULL);
+ init(MockMatcher *self, I32Array *doc_ids, Blob *scores = NULL);
public void
Destroy(MockMatcher *self);
http://git-wip-us.apache.org/repos/asf/lucy/blob/5651dd32/perl/lib/LucyX/Search/MockMatcher.pm
----------------------------------------------------------------------
diff --git a/perl/lib/LucyX/Search/MockMatcher.pm b/perl/lib/LucyX/Search/MockMatcher.pm
index f33a3af..0937f11 100644
--- a/perl/lib/LucyX/Search/MockMatcher.pm
+++ b/perl/lib/LucyX/Search/MockMatcher.pm
@@ -30,7 +30,7 @@ sub new {
if ( ref( $args{scores} ) eq 'ARRAY' ) {
confess("Mismatch between scores and doc_ids array sizes")
unless scalar @{ $args{scores} } == $size;
- $scores = Clownfish::ByteBuf->new(
+ $scores = Clownfish::Blob->new(
pack( "f$size", @{ $args{scores} } ) );
}
[8/9] lucy git commit: Make Slurp_File return a Blob
Posted by nw...@apache.org.
Make Slurp_File return a Blob
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/4e18b58a
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/4e18b58a
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/4e18b58a
Branch: refs/heads/master
Commit: 4e18b58a911b495e61a5311610efa035344e8a1d
Parents: c6ccfdb
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Tue May 5 11:07:05 2015 +0200
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Tue May 5 11:24:53 2015 +0200
----------------------------------------------------------------------
core/Lucy/Store/Folder.c | 7 ++++---
core/Lucy/Store/Folder.cfh | 2 +-
core/Lucy/Test/Store/TestFolder.c | 5 +++--
3 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/4e18b58a/core/Lucy/Store/Folder.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Folder.c b/core/Lucy/Store/Folder.c
index d724da0..244446a 100644
--- a/core/Lucy/Store/Folder.c
+++ b/core/Lucy/Store/Folder.c
@@ -21,6 +21,7 @@
#include "charmony.h"
+#include "Clownfish/Blob.h"
#include "Lucy/Store/Folder.h"
#include "Lucy/Store/CompoundFileReader.h"
#include "Lucy/Store/CompoundFileWriter.h"
@@ -363,10 +364,10 @@ Folder_List_R_IMP(Folder *self, String *path) {
return list;
}
-ByteBuf*
+Blob*
Folder_Slurp_File_IMP(Folder *self, String *path) {
InStream *instream = Folder_Open_In(self, path);
- ByteBuf *retval = NULL;
+ Blob *retval = NULL;
if (!instream) {
RETHROW(INCREF(Err_get_error()));
@@ -385,7 +386,7 @@ Folder_Slurp_File_IMP(Folder *self, String *path) {
char *ptr = (char*)MALLOCATE((size_t)size + 1);
InStream_Read_Bytes(instream, ptr, size);
ptr[size] = '\0';
- retval = BB_new_steal_bytes(ptr, size, size + 1);
+ retval = Blob_new_steal(ptr, size);
InStream_Close(instream);
DECREF(instream);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/4e18b58a/core/Lucy/Store/Folder.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/Folder.cfh b/core/Lucy/Store/Folder.cfh
index a2c1bf0..778b12d 100644
--- a/core/Lucy/Store/Folder.cfh
+++ b/core/Lucy/Store/Folder.cfh
@@ -170,7 +170,7 @@ public abstract class Lucy::Store::Folder inherits Clownfish::Obj {
* @param path A relative filepath.
* @param return the file's contents.
*/
- public incremented ByteBuf*
+ public incremented Blob*
Slurp_File(Folder *self, String *path);
/** Collapse the contents of the directory into a compound file.
http://git-wip-us.apache.org/repos/asf/lucy/blob/4e18b58a/core/Lucy/Test/Store/TestFolder.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Test/Store/TestFolder.c b/core/Lucy/Test/Store/TestFolder.c
index 34d683d..f5b0464 100644
--- a/core/Lucy/Test/Store/TestFolder.c
+++ b/core/Lucy/Test/Store/TestFolder.c
@@ -18,6 +18,7 @@
#define TESTLUCY_USE_SHORT_NAMES
#include "Lucy/Util/ToolSet.h"
+#include "Clownfish/Blob.h"
#include "Clownfish/TestHarness/TestBatchRunner.h"
#include "Lucy/Test.h"
#include "Lucy/Test/Store/TestFolder.h"
@@ -506,13 +507,13 @@ test_Slurp_File(TestBatchRunner *runner) {
Folder *folder = (Folder*)RAMFolder_new(NULL);
FileHandle *fh = Folder_Open_FileHandle(folder, foo,
FH_CREATE | FH_WRITE_ONLY);
- ByteBuf *contents;
+ Blob *contents;
FH_Write(fh, "stuff", 5);
FH_Close(fh);
DECREF(fh);
contents = Folder_Slurp_File(folder, foo);
- TEST_TRUE(runner, BB_Equals_Bytes(contents, "stuff", 5), "Slurp_File");
+ TEST_TRUE(runner, Blob_Equals_Bytes(contents, "stuff", 5), "Slurp_File");
DECREF(contents);
DECREF(folder);
[6/9] lucy git commit: Switch TestSortExternal over to Blob
Posted by nw...@apache.org.
Switch TestSortExternal over to Blob
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/2f7df540
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/2f7df540
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/2f7df540
Branch: refs/heads/master
Commit: 2f7df540f27fae676082332612718f9da79bc7bb
Parents: 32e5dc4
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Mon May 4 17:51:53 2015 +0200
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Tue May 5 11:23:05 2015 +0200
----------------------------------------------------------------------
core/Lucy/Test/Util/TestSortExternal.c | 215 ++++++++++++++--------------
core/Lucy/Util/BBSortEx.c | 200 --------------------------
core/Lucy/Util/BBSortEx.cfh | 67 ---------
core/Lucy/Util/BlobSortEx.c | 201 ++++++++++++++++++++++++++
core/Lucy/Util/BlobSortEx.cfh | 67 +++++++++
perl/t/015-sort_external.t | 54 +++----
6 files changed, 403 insertions(+), 401 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/2f7df540/core/Lucy/Test/Util/TestSortExternal.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Test/Util/TestSortExternal.c b/core/Lucy/Test/Util/TestSortExternal.c
index 059ccca..e9bbba9 100644
--- a/core/Lucy/Test/Util/TestSortExternal.c
+++ b/core/Lucy/Test/Util/TestSortExternal.c
@@ -22,17 +22,18 @@
#include "Lucy/Util/ToolSet.h"
#include "Lucy/Test/Util/TestSortExternal.h"
+#include "Clownfish/Blob.h"
#include "Clownfish/TestHarness/TestBatchRunner.h"
-#include "Lucy/Util/BBSortEx.h"
+#include "Lucy/Util/BlobSortEx.h"
#include "Lucy/Util/SortExternal.h"
-static ByteBuf *a_bb;
-static ByteBuf *b_bb;
-static ByteBuf *c_bb;
-static ByteBuf *d_bb;
-static ByteBuf *x_bb;
-static ByteBuf *y_bb;
-static ByteBuf *z_bb;
+static Blob *a_blob;
+static Blob *b_blob;
+static Blob *c_blob;
+static Blob *d_blob;
+static Blob *x_blob;
+static Blob *y_blob;
+static Blob *z_blob;
TestSortExternal*
TestSortExternal_new() {
@@ -40,78 +41,78 @@ TestSortExternal_new() {
}
static void
-S_init_bytebufs() {
- a_bb = BB_new_bytes("a", 1);
- b_bb = BB_new_bytes("b", 1);
- c_bb = BB_new_bytes("c", 1);
- d_bb = BB_new_bytes("d", 1);
- x_bb = BB_new_bytes("x", 1);
- y_bb = BB_new_bytes("y", 1);
- z_bb = BB_new_bytes("z", 1);
+S_init_blobs() {
+ a_blob = Blob_new("a", 1);
+ b_blob = Blob_new("b", 1);
+ c_blob = Blob_new("c", 1);
+ d_blob = Blob_new("d", 1);
+ x_blob = Blob_new("x", 1);
+ y_blob = Blob_new("y", 1);
+ z_blob = Blob_new("z", 1);
}
static void
-S_destroy_bytebufs() {
- DECREF(a_bb);
- DECREF(b_bb);
- DECREF(c_bb);
- DECREF(d_bb);
- DECREF(x_bb);
- DECREF(y_bb);
- DECREF(z_bb);
+S_destroy_blobs() {
+ DECREF(a_blob);
+ DECREF(b_blob);
+ DECREF(c_blob);
+ DECREF(d_blob);
+ DECREF(x_blob);
+ DECREF(y_blob);
+ DECREF(z_blob);
}
static void
test_bbsortex(TestBatchRunner *runner) {
- BBSortEx *sortex = BBSortEx_new(4, NULL);
+ BlobSortEx *sortex = BlobSortEx_new(4, NULL);
- BBSortEx_Feed(sortex, INCREF(c_bb));
- TEST_INT_EQ(runner, BBSortEx_Buffer_Count(sortex), 1,
+ BlobSortEx_Feed(sortex, INCREF(c_blob));
+ TEST_INT_EQ(runner, BlobSortEx_Buffer_Count(sortex), 1,
"feed elem into cache");
- BBSortEx_Feed(sortex, INCREF(b_bb));
- BBSortEx_Feed(sortex, INCREF(d_bb));
- BBSortEx_Sort_Buffer(sortex);
+ BlobSortEx_Feed(sortex, INCREF(b_blob));
+ BlobSortEx_Feed(sortex, INCREF(d_blob));
+ BlobSortEx_Sort_Buffer(sortex);
{
- Vector *cache = BBSortEx_Peek_Cache(sortex);
+ Vector *cache = BlobSortEx_Peek_Cache(sortex);
Vector *wanted = Vec_new(3);
- Vec_Push(wanted, INCREF(b_bb));
- Vec_Push(wanted, INCREF(c_bb));
- Vec_Push(wanted, INCREF(d_bb));
+ Vec_Push(wanted, INCREF(b_blob));
+ Vec_Push(wanted, INCREF(c_blob));
+ Vec_Push(wanted, INCREF(d_blob));
TEST_TRUE(runner, Vec_Equals(cache, (Obj*)wanted), "sort cache");
DECREF(wanted);
DECREF(cache);
}
- BBSortEx_Feed(sortex, INCREF(a_bb));
- TEST_INT_EQ(runner, BBSortEx_Buffer_Count(sortex), 0,
+ BlobSortEx_Feed(sortex, INCREF(a_blob));
+ TEST_INT_EQ(runner, BlobSortEx_Buffer_Count(sortex), 0,
"cache flushed automatically when mem_thresh crossed");
- TEST_INT_EQ(runner, BBSortEx_Get_Num_Runs(sortex), 1, "run added");
+ TEST_INT_EQ(runner, BlobSortEx_Get_Num_Runs(sortex), 1, "run added");
Vector *external = Vec_new(3);
- Vec_Push(external, INCREF(x_bb));
- Vec_Push(external, INCREF(y_bb));
- Vec_Push(external, INCREF(z_bb));
- BBSortEx *run = BBSortEx_new(0x1000000, external);
- BBSortEx_Add_Run(sortex, (SortExternal*)run);
- BBSortEx_Flip(sortex);
+ Vec_Push(external, INCREF(x_blob));
+ Vec_Push(external, INCREF(y_blob));
+ Vec_Push(external, INCREF(z_blob));
+ BlobSortEx *run = BlobSortEx_new(0x1000000, external);
+ BlobSortEx_Add_Run(sortex, (SortExternal*)run);
+ BlobSortEx_Flip(sortex);
{
Vector *got = Vec_new(7);
Obj *object;
- while (NULL != (object = BBSortEx_Fetch(sortex))) {
+ while (NULL != (object = BlobSortEx_Fetch(sortex))) {
Vec_Push(got, object);
}
Vector *wanted = Vec_new(7);
- Vec_Push(wanted, INCREF(a_bb));
- Vec_Push(wanted, INCREF(b_bb));
- Vec_Push(wanted, INCREF(c_bb));
- Vec_Push(wanted, INCREF(d_bb));
- Vec_Push(wanted, INCREF(x_bb));
- Vec_Push(wanted, INCREF(y_bb));
- Vec_Push(wanted, INCREF(z_bb));
+ Vec_Push(wanted, INCREF(a_blob));
+ Vec_Push(wanted, INCREF(b_blob));
+ Vec_Push(wanted, INCREF(c_blob));
+ Vec_Push(wanted, INCREF(d_blob));
+ Vec_Push(wanted, INCREF(x_blob));
+ Vec_Push(wanted, INCREF(y_blob));
+ Vec_Push(wanted, INCREF(z_blob));
TEST_TRUE(runner, Vec_Equals(got, (Obj*)wanted), "Add_Run");
@@ -125,26 +126,26 @@ test_bbsortex(TestBatchRunner *runner) {
static void
test_clear_buffer(TestBatchRunner *runner) {
- BBSortEx *sortex = BBSortEx_new(4, NULL);
+ BlobSortEx *sortex = BlobSortEx_new(4, NULL);
- BBSortEx_Feed(sortex, INCREF(c_bb));
- BBSortEx_Clear_Buffer(sortex);
- TEST_INT_EQ(runner, BBSortEx_Buffer_Count(sortex), 0, "Clear_Buffer");
+ BlobSortEx_Feed(sortex, INCREF(c_blob));
+ BlobSortEx_Clear_Buffer(sortex);
+ TEST_INT_EQ(runner, BlobSortEx_Buffer_Count(sortex), 0, "Clear_Buffer");
- BBSortEx_Feed(sortex, INCREF(b_bb));
- BBSortEx_Feed(sortex, INCREF(a_bb));
- BBSortEx_Flush(sortex);
- BBSortEx_Flip(sortex);
- Obj *object = BBSortEx_Peek(sortex);
- TEST_TRUE(runner, BB_Equals(a_bb, object), "Peek");
+ BlobSortEx_Feed(sortex, INCREF(b_blob));
+ BlobSortEx_Feed(sortex, INCREF(a_blob));
+ BlobSortEx_Flush(sortex);
+ BlobSortEx_Flip(sortex);
+ Obj *object = BlobSortEx_Peek(sortex);
+ TEST_TRUE(runner, Blob_Equals(a_blob, object), "Peek");
Vector *got = Vec_new(2);
- while (NULL != (object = BBSortEx_Fetch(sortex))) {
+ while (NULL != (object = BlobSortEx_Fetch(sortex))) {
Vec_Push(got, object);
}
Vector *wanted = Vec_new(2);
- Vec_Push(wanted, INCREF(a_bb));
- Vec_Push(wanted, INCREF(b_bb));
+ Vec_Push(wanted, INCREF(a_blob));
+ Vec_Push(wanted, INCREF(b_blob));
TEST_TRUE(runner, Vec_Equals(got, (Obj*)wanted),
"elements cleared via Clear_Buffer truly cleared");
@@ -154,32 +155,32 @@ test_clear_buffer(TestBatchRunner *runner) {
}
static void
-S_test_sort(TestBatchRunner *runner, Vector *bytebufs, uint32_t mem_thresh,
+S_test_sort(TestBatchRunner *runner, Vector *blobs, uint32_t mem_thresh,
const char *test_name) {
- int size = (int)Vec_Get_Size(bytebufs);
- BBSortEx *sortex = BBSortEx_new(mem_thresh, NULL);
- ByteBuf **shuffled = (ByteBuf**)MALLOCATE(size * sizeof(ByteBuf*));
+ int size = (int)Vec_Get_Size(blobs);
+ BlobSortEx *sortex = BlobSortEx_new(mem_thresh, NULL);
+ Blob **shuffled = (Blob**)MALLOCATE(size * sizeof(Blob*));
for (int i = 0; i < size; ++i) {
- shuffled[i] = (ByteBuf*)CERTIFY(Vec_Fetch(bytebufs, i), BYTEBUF);
+ shuffled[i] = (Blob*)CERTIFY(Vec_Fetch(blobs, i), BLOB);
}
for (int i = size - 1; i > 0; --i) {
int shuffle_pos = rand() % (i + 1);
- ByteBuf *temp = shuffled[shuffle_pos];
+ Blob *temp = shuffled[shuffle_pos];
shuffled[shuffle_pos] = shuffled[i];
shuffled[i] = temp;
}
for (int i = 0; i < size; ++i) {
- BBSortEx_Feed(sortex, INCREF(shuffled[i]));
+ BlobSortEx_Feed(sortex, INCREF(shuffled[i]));
}
- BBSortEx_Flip(sortex);
+ BlobSortEx_Flip(sortex);
Vector *got = Vec_new(size);
Obj *object;
- while (NULL != (object = BBSortEx_Fetch(sortex))) {
+ while (NULL != (object = BlobSortEx_Fetch(sortex))) {
Vec_Push(got, object);
}
- TEST_TRUE(runner, Vec_Equals(got, (Obj*)bytebufs), test_name);
+ TEST_TRUE(runner, Vec_Equals(got, (Obj*)blobs), test_name);
FREEMEM(shuffled);
DECREF(got);
@@ -190,18 +191,18 @@ static void
S_test_sort_letters(TestBatchRunner *runner, const char *letters,
uint32_t mem_thresh, const char *test_name) {
size_t num_letters = strlen(letters);
- Vector *bytebufs = Vec_new(num_letters);
+ Vector *blobs = Vec_new(num_letters);
for (size_t i = 0; i < num_letters; ++i) {
char ch = letters[i];
size_t size = ch == '_' ? 0 : 1;
- ByteBuf *bytebuf = BB_new_bytes(&ch, size);
- Vec_Push(bytebufs, (Obj*)bytebuf);
+ Blob *blob = Blob_new(&ch, size);
+ Vec_Push(blobs, (Obj*)blob);
}
- S_test_sort(runner, bytebufs, mem_thresh, test_name);
+ S_test_sort(runner, blobs, mem_thresh, test_name);
- DECREF(bytebufs);
+ DECREF(blobs);
}
static void
@@ -220,9 +221,9 @@ test_sort_letters(TestBatchRunner *runner) {
static void
test_sort_nothing(TestBatchRunner *runner) {
- BBSortEx *sortex = BBSortEx_new(0x1000000, NULL);
- BBSortEx_Flip(sortex);
- TEST_TRUE(runner, BBSortEx_Fetch(sortex) == NULL,
+ BlobSortEx *sortex = BlobSortEx_new(0x1000000, NULL);
+ BlobSortEx_Flip(sortex);
+ TEST_TRUE(runner, BlobSortEx_Fetch(sortex) == NULL,
"Sorting nothing returns undef");
DECREF(sortex);
}
@@ -230,7 +231,7 @@ test_sort_nothing(TestBatchRunner *runner) {
static void
test_sort_packed_ints(TestBatchRunner *runner) {
size_t num_ints = 11001;
- Vector *bytebufs = Vec_new(num_ints);
+ Vector *blobs = Vec_new(num_ints);
for (uint32_t i = 0; i < num_ints; ++i) {
char buf[4];
@@ -238,19 +239,19 @@ test_sort_packed_ints(TestBatchRunner *runner) {
buf[1] = i >> 16;
buf[2] = i >> 8;
buf[3] = i;
- ByteBuf *bytebuf = BB_new_bytes(&buf, 4);
- Vec_Push(bytebufs, (Obj*)bytebuf);
+ Blob *blob = Blob_new(buf, 4);
+ Vec_Push(blobs, (Obj*)blob);
}
- S_test_sort(runner, bytebufs, 5000, "Sorting packed integers...");
+ S_test_sort(runner, blobs, 5000, "Sorting packed integers...");
- DECREF(bytebufs);
+ DECREF(blobs);
}
static void
test_sort_random_strings(TestBatchRunner *runner) {
size_t num_strings = 1001;
- Vector *bytebufs = Vec_new(num_strings);
+ Vector *blobs = Vec_new(num_strings);
for (uint32_t i = 0; i < num_strings; ++i) {
char buf[1201];
@@ -258,15 +259,15 @@ test_sort_random_strings(TestBatchRunner *runner) {
for (int i = 0; i < size; ++i) {
buf[i] = rand();
}
- ByteBuf *bytebuf = BB_new_bytes(&buf, size);
- Vec_Push(bytebufs, (Obj*)bytebuf);
+ Blob *blob = Blob_new(buf, size);
+ Vec_Push(blobs, (Obj*)blob);
}
- Vec_Sort(bytebufs);
- S_test_sort(runner, bytebufs, 15000,
+ Vec_Sort(blobs);
+ S_test_sort(runner, blobs, 15000,
"Random binary strings of random length");
- DECREF(bytebufs);
+ DECREF(blobs);
}
static void
@@ -274,27 +275,27 @@ test_run(TestBatchRunner *runner) {
Vector *letters = Vec_new(26);
for (int i = 0; i < 26; ++i) {
char ch = 'a' + i;
- ByteBuf *bytebuf = BB_new_bytes(&ch, 1);
- Vec_Push(letters, (Obj*)bytebuf);
+ Blob *blob = Blob_new(&ch, 1);
+ Vec_Push(letters, (Obj*)blob);
}
- BBSortEx *run = BBSortEx_new(0x1000000, letters);
- BBSortEx_Set_Mem_Thresh(run, 5);
+ BlobSortEx *run = BlobSortEx_new(0x1000000, letters);
+ BlobSortEx_Set_Mem_Thresh(run, 5);
- BBSortEx_Refill(run);
- TEST_INT_EQ(runner, BBSortEx_Buffer_Count(run), 5,
+ BlobSortEx_Refill(run);
+ TEST_INT_EQ(runner, BlobSortEx_Buffer_Count(run), 5,
"Refill doesn't exceed memory threshold");
- Obj *endpost = BBSortEx_Peek_Last(run);
- ByteBuf *wanted = BB_new_bytes("e", 1);
- TEST_TRUE(runner, BB_Equals(wanted, endpost), "Peek_Last");
+ Obj *endpost = BlobSortEx_Peek_Last(run);
+ Blob *wanted = Blob_new("e", 1);
+ TEST_TRUE(runner, Blob_Equals(wanted, endpost), "Peek_Last");
Vector *elems = Vec_new(26);
do {
- while (BBSortEx_Buffer_Count(run) > 0) {
- Obj *object = BBSortEx_Fetch(run);
+ while (BlobSortEx_Buffer_Count(run) > 0) {
+ Obj *object = BlobSortEx_Fetch(run);
Vec_Push(elems, object);
}
- } while (BBSortEx_Refill(run) > 0);
+ } while (BlobSortEx_Refill(run) > 0);
TEST_TRUE(runner, Vec_Equals(elems, (Obj*)letters), "retrieve all elems");
DECREF(elems);
@@ -309,7 +310,7 @@ TestSortExternal_Run_IMP(TestSortExternal *self, TestBatchRunner *runner) {
TestBatchRunner_Plan(runner, (TestBatch*)self, 19);
srand((unsigned int)time((time_t*)NULL));
- S_init_bytebufs();
+ S_init_blobs();
test_bbsortex(runner);
test_clear_buffer(runner);
test_sort_letters(runner);
@@ -317,7 +318,7 @@ TestSortExternal_Run_IMP(TestSortExternal *self, TestBatchRunner *runner) {
test_sort_packed_ints(runner);
test_sort_random_strings(runner);
test_run(runner);
- S_destroy_bytebufs();
+ S_destroy_blobs();
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/2f7df540/core/Lucy/Util/BBSortEx.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/BBSortEx.c b/core/Lucy/Util/BBSortEx.c
deleted file mode 100644
index 5a9a3e0..0000000
--- a/core/Lucy/Util/BBSortEx.c
+++ /dev/null
@@ -1,200 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#define C_LUCY_BBSORTEX
-#define LUCY_USE_SHORT_NAMES
-#include "Lucy/Util/ToolSet.h"
-
-#include "Lucy/Util/BBSortEx.h"
-
-#include "Lucy/Index/Segment.h"
-#include "Lucy/Store/InStream.h"
-#include "Lucy/Store/Folder.h"
-#include "Lucy/Store/OutStream.h"
-
-BBSortEx*
-BBSortEx_new(uint32_t mem_threshold, Vector *external) {
- BBSortEx *self = (BBSortEx*)Class_Make_Obj(BBSORTEX);
- return BBSortEx_init(self, mem_threshold, external);
-}
-
-BBSortEx*
-BBSortEx_init(BBSortEx *self, uint32_t mem_threshold, Vector *external) {
- SortEx_init((SortExternal*)self);
- BBSortExIVARS *const ivars = BBSortEx_IVARS(self);
- ivars->external_tick = 0;
- ivars->external = (Vector*)INCREF(external);
- ivars->mem_consumed = 0;
- BBSortEx_Set_Mem_Thresh(self, mem_threshold);
- return self;
-}
-
-void
-BBSortEx_Destroy_IMP(BBSortEx *self) {
- BBSortExIVARS *const ivars = BBSortEx_IVARS(self);
- DECREF(ivars->external);
- SUPER_DESTROY(self, BBSORTEX);
-}
-
-void
-BBSortEx_Clear_Buffer_IMP(BBSortEx *self) {
- BBSortExIVARS *const ivars = BBSortEx_IVARS(self);
- ivars->mem_consumed = 0;
- BBSortEx_Clear_Buffer_t super_clear_buffer
- = SUPER_METHOD_PTR(BBSORTEX, LUCY_BBSortEx_Clear_Buffer);
- super_clear_buffer(self);
-}
-
-void
-BBSortEx_Feed_IMP(BBSortEx *self, Obj *item) {
- BBSortExIVARS *const ivars = BBSortEx_IVARS(self);
- BBSortEx_Feed_t super_feed
- = SUPER_METHOD_PTR(BBSORTEX, LUCY_BBSortEx_Feed);
- super_feed(self, item);
-
- // Flush() if necessary.
- ByteBuf *bytebuf = (ByteBuf*)CERTIFY(item, BYTEBUF);
- ivars->mem_consumed += BB_Get_Size(bytebuf);
- if (ivars->mem_consumed >= ivars->mem_thresh) {
- BBSortEx_Flush(self);
- }
-}
-
-void
-BBSortEx_Flush_IMP(BBSortEx *self) {
- BBSortExIVARS *const ivars = BBSortEx_IVARS(self);
- uint32_t buf_count = ivars->buf_max - ivars->buf_tick;
- Obj **buffer = ivars->buffer;
- Vector *elems;
-
- if (!buf_count) { return; }
- else { elems = Vec_new(buf_count); }
-
- // Sort, then create a new run.
- BBSortEx_Sort_Buffer(self);
- for (uint32_t i = ivars->buf_tick; i < ivars->buf_max; i++) {
- Vec_Push(elems, buffer[i]);
- }
- BBSortEx *run = BBSortEx_new(0, elems);
- DECREF(elems);
- BBSortEx_Add_Run(self, (SortExternal*)run);
-
- // Blank the buffer vars.
- ivars->buf_tick += buf_count;
- BBSortEx_Clear_Buffer(self);
-}
-
-uint32_t
-BBSortEx_Refill_IMP(BBSortEx *self) {
- BBSortExIVARS *const ivars = BBSortEx_IVARS(self);
-
- // Make sure buffer is empty, then set buffer tick vars.
- if (ivars->buf_max - ivars->buf_tick > 0) {
- THROW(ERR, "Refill called but buffer contains %u32 items",
- ivars->buf_max - ivars->buf_tick);
- }
- ivars->buf_tick = 0;
- ivars->buf_max = 0;
-
- // Read in elements.
- while (1) {
- ByteBuf *elem = NULL;
-
- if (ivars->mem_consumed >= ivars->mem_thresh) {
- ivars->mem_consumed = 0;
- break;
- }
- else if (ivars->external_tick >= Vec_Get_Size(ivars->external)) {
- break;
- }
- else {
- elem = (ByteBuf*)Vec_Fetch(ivars->external, ivars->external_tick);
- ivars->external_tick++;
- // Should be + sizeof(ByteBuf), but that's ok.
- ivars->mem_consumed += BB_Get_Size(elem);
- }
-
- if (ivars->buf_max == ivars->buf_cap) {
- BBSortEx_Grow_Buffer(self,
- Memory_oversize(ivars->buf_max + 1,
- sizeof(Obj*)));
- }
- ivars->buffer[ivars->buf_max++] = INCREF(elem);
- }
-
- return ivars->buf_max;
-}
-
-void
-BBSortEx_Flip_IMP(BBSortEx *self) {
- BBSortExIVARS *const ivars = BBSortEx_IVARS(self);
- uint32_t run_mem_thresh = 65536;
-
- BBSortEx_Flush(self);
-
- // Recalculate the approximate mem allowed for each run.
- uint32_t num_runs = Vec_Get_Size(ivars->runs);
- if (num_runs) {
- run_mem_thresh = (ivars->mem_thresh / 2) / num_runs;
- if (run_mem_thresh < 65536) {
- run_mem_thresh = 65536;
- }
- }
-
- for (uint32_t i = 0; i < num_runs; i++) {
- BBSortEx *run = (BBSortEx*)Vec_Fetch(ivars->runs, i);
- BBSortEx_Set_Mem_Thresh(run, run_mem_thresh);
- }
-
- // OK to fetch now.
- ivars->flipped = true;
-}
-
-int
-BBSortEx_Compare_IMP(BBSortEx *self, void *va, void *vb) {
- UNUSED_VAR(self);
- return BB_compare((ByteBuf**)va, (ByteBuf**)vb);
-}
-
-Vector*
-BBSortEx_Peek_Cache_IMP(BBSortEx *self) {
- BBSortExIVARS *const ivars = BBSortEx_IVARS(self);
- uint32_t count = ivars->buf_max - ivars->buf_tick;
- Obj **buffer = ivars->buffer;
- Vector *retval = Vec_new(count);
-
- for (uint32_t i = ivars->buf_tick; i < ivars->buf_max; ++i) {
- Vec_Push(retval, INCREF(buffer[i]));
- }
-
- return retval;
-}
-
-Obj*
-BBSortEx_Peek_Last_IMP(BBSortEx *self) {
- BBSortExIVARS *const ivars = BBSortEx_IVARS(self);
- uint32_t count = ivars->buf_max - ivars->buf_tick;
- if (count == 0) { return NULL; }
- return INCREF(ivars->buffer[count-1]);
-}
-
-uint32_t
-BBSortEx_Get_Num_Runs_IMP(BBSortEx *self) {
- BBSortExIVARS *const ivars = BBSortEx_IVARS(self);
- return Vec_Get_Size(ivars->runs);
-}
-
-
http://git-wip-us.apache.org/repos/asf/lucy/blob/2f7df540/core/Lucy/Util/BBSortEx.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/BBSortEx.cfh b/core/Lucy/Util/BBSortEx.cfh
deleted file mode 100644
index eaac4c6..0000000
--- a/core/Lucy/Util/BBSortEx.cfh
+++ /dev/null
@@ -1,67 +0,0 @@
-/* Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-parcel Lucy;
-
-/** SortExternal for ByteBufs.
- */
-
-class Lucy::Util::BBSortEx
- inherits Lucy::Util::SortExternal {
-
- Vector *external;
- uint32_t external_tick;
- uint32_t mem_consumed;
-
- inert BBSortEx*
- new(uint32_t mem_thresh = 0x1000000, Vector *external = NULL);
-
- inert BBSortEx*
- init(BBSortEx *self, uint32_t mem_thresh = 0x1000000,
- Vector *external = NULL);
-
- void
- Feed(BBSortEx *self, decremented Obj *item);
-
- void
- Flush(BBSortEx *self);
-
- uint32_t
- Refill(BBSortEx *self);
-
- void
- Clear_Buffer(BBSortEx *self);
-
- void
- Flip(BBSortEx *self);
-
- int
- Compare(BBSortEx *self, void *va, void *vb);
-
- incremented Vector*
- Peek_Cache(BBSortEx *self);
-
- incremented nullable Obj*
- Peek_Last(BBSortEx *self);
-
- uint32_t
- Get_Num_Runs(BBSortEx *self);
-
- public void
- Destroy(BBSortEx *self);
-}
-
-
http://git-wip-us.apache.org/repos/asf/lucy/blob/2f7df540/core/Lucy/Util/BlobSortEx.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/BlobSortEx.c b/core/Lucy/Util/BlobSortEx.c
new file mode 100644
index 0000000..6d41488
--- /dev/null
+++ b/core/Lucy/Util/BlobSortEx.c
@@ -0,0 +1,201 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_BLOBSORTEX
+#define LUCY_USE_SHORT_NAMES
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Util/BlobSortEx.h"
+
+#include "Clownfish/Blob.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/OutStream.h"
+
+BlobSortEx*
+BlobSortEx_new(uint32_t mem_threshold, Vector *external) {
+ BlobSortEx *self = (BlobSortEx*)Class_Make_Obj(BLOBSORTEX);
+ return BlobSortEx_init(self, mem_threshold, external);
+}
+
+BlobSortEx*
+BlobSortEx_init(BlobSortEx *self, uint32_t mem_threshold, Vector *external) {
+ SortEx_init((SortExternal*)self);
+ BlobSortExIVARS *const ivars = BlobSortEx_IVARS(self);
+ ivars->external_tick = 0;
+ ivars->external = (Vector*)INCREF(external);
+ ivars->mem_consumed = 0;
+ BlobSortEx_Set_Mem_Thresh(self, mem_threshold);
+ return self;
+}
+
+void
+BlobSortEx_Destroy_IMP(BlobSortEx *self) {
+ BlobSortExIVARS *const ivars = BlobSortEx_IVARS(self);
+ DECREF(ivars->external);
+ SUPER_DESTROY(self, BLOBSORTEX);
+}
+
+void
+BlobSortEx_Clear_Buffer_IMP(BlobSortEx *self) {
+ BlobSortExIVARS *const ivars = BlobSortEx_IVARS(self);
+ ivars->mem_consumed = 0;
+ BlobSortEx_Clear_Buffer_t super_clear_buffer
+ = SUPER_METHOD_PTR(BLOBSORTEX, LUCY_BlobSortEx_Clear_Buffer);
+ super_clear_buffer(self);
+}
+
+void
+BlobSortEx_Feed_IMP(BlobSortEx *self, Obj *item) {
+ BlobSortExIVARS *const ivars = BlobSortEx_IVARS(self);
+ BlobSortEx_Feed_t super_feed
+ = SUPER_METHOD_PTR(BLOBSORTEX, LUCY_BlobSortEx_Feed);
+ super_feed(self, item);
+
+ // Flush() if necessary.
+ Blob *blob = (Blob*)CERTIFY(item, BLOB);
+ ivars->mem_consumed += Blob_Get_Size(blob);
+ if (ivars->mem_consumed >= ivars->mem_thresh) {
+ BlobSortEx_Flush(self);
+ }
+}
+
+void
+BlobSortEx_Flush_IMP(BlobSortEx *self) {
+ BlobSortExIVARS *const ivars = BlobSortEx_IVARS(self);
+ uint32_t buf_count = ivars->buf_max - ivars->buf_tick;
+ Obj **buffer = ivars->buffer;
+ Vector *elems;
+
+ if (!buf_count) { return; }
+ else { elems = Vec_new(buf_count); }
+
+ // Sort, then create a new run.
+ BlobSortEx_Sort_Buffer(self);
+ for (uint32_t i = ivars->buf_tick; i < ivars->buf_max; i++) {
+ Vec_Push(elems, buffer[i]);
+ }
+ BlobSortEx *run = BlobSortEx_new(0, elems);
+ DECREF(elems);
+ BlobSortEx_Add_Run(self, (SortExternal*)run);
+
+ // Blank the buffer vars.
+ ivars->buf_tick += buf_count;
+ BlobSortEx_Clear_Buffer(self);
+}
+
+uint32_t
+BlobSortEx_Refill_IMP(BlobSortEx *self) {
+ BlobSortExIVARS *const ivars = BlobSortEx_IVARS(self);
+
+ // Make sure buffer is empty, then set buffer tick vars.
+ if (ivars->buf_max - ivars->buf_tick > 0) {
+ THROW(ERR, "Refill called but buffer contains %u32 items",
+ ivars->buf_max - ivars->buf_tick);
+ }
+ ivars->buf_tick = 0;
+ ivars->buf_max = 0;
+
+ // Read in elements.
+ while (1) {
+ Blob *elem = NULL;
+
+ if (ivars->mem_consumed >= ivars->mem_thresh) {
+ ivars->mem_consumed = 0;
+ break;
+ }
+ else if (ivars->external_tick >= Vec_Get_Size(ivars->external)) {
+ break;
+ }
+ else {
+ elem = (Blob*)Vec_Fetch(ivars->external, ivars->external_tick);
+ ivars->external_tick++;
+ // Should be + sizeof(Blob), but that's ok.
+ ivars->mem_consumed += Blob_Get_Size(elem);
+ }
+
+ if (ivars->buf_max == ivars->buf_cap) {
+ BlobSortEx_Grow_Buffer(self,
+ Memory_oversize(ivars->buf_max + 1,
+ sizeof(Obj*)));
+ }
+ ivars->buffer[ivars->buf_max++] = INCREF(elem);
+ }
+
+ return ivars->buf_max;
+}
+
+void
+BlobSortEx_Flip_IMP(BlobSortEx *self) {
+ BlobSortExIVARS *const ivars = BlobSortEx_IVARS(self);
+ uint32_t run_mem_thresh = 65536;
+
+ BlobSortEx_Flush(self);
+
+ // Recalculate the approximate mem allowed for each run.
+ uint32_t num_runs = Vec_Get_Size(ivars->runs);
+ if (num_runs) {
+ run_mem_thresh = (ivars->mem_thresh / 2) / num_runs;
+ if (run_mem_thresh < 65536) {
+ run_mem_thresh = 65536;
+ }
+ }
+
+ for (uint32_t i = 0; i < num_runs; i++) {
+ BlobSortEx *run = (BlobSortEx*)Vec_Fetch(ivars->runs, i);
+ BlobSortEx_Set_Mem_Thresh(run, run_mem_thresh);
+ }
+
+ // OK to fetch now.
+ ivars->flipped = true;
+}
+
+int
+BlobSortEx_Compare_IMP(BlobSortEx *self, void *va, void *vb) {
+ UNUSED_VAR(self);
+ return Blob_compare((Blob**)va, (Blob**)vb);
+}
+
+Vector*
+BlobSortEx_Peek_Cache_IMP(BlobSortEx *self) {
+ BlobSortExIVARS *const ivars = BlobSortEx_IVARS(self);
+ uint32_t count = ivars->buf_max - ivars->buf_tick;
+ Obj **buffer = ivars->buffer;
+ Vector *retval = Vec_new(count);
+
+ for (uint32_t i = ivars->buf_tick; i < ivars->buf_max; ++i) {
+ Vec_Push(retval, INCREF(buffer[i]));
+ }
+
+ return retval;
+}
+
+Obj*
+BlobSortEx_Peek_Last_IMP(BlobSortEx *self) {
+ BlobSortExIVARS *const ivars = BlobSortEx_IVARS(self);
+ uint32_t count = ivars->buf_max - ivars->buf_tick;
+ if (count == 0) { return NULL; }
+ return INCREF(ivars->buffer[count-1]);
+}
+
+uint32_t
+BlobSortEx_Get_Num_Runs_IMP(BlobSortEx *self) {
+ BlobSortExIVARS *const ivars = BlobSortEx_IVARS(self);
+ return Vec_Get_Size(ivars->runs);
+}
+
+
http://git-wip-us.apache.org/repos/asf/lucy/blob/2f7df540/core/Lucy/Util/BlobSortEx.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/BlobSortEx.cfh b/core/Lucy/Util/BlobSortEx.cfh
new file mode 100644
index 0000000..47cef86
--- /dev/null
+++ b/core/Lucy/Util/BlobSortEx.cfh
@@ -0,0 +1,67 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** SortExternal for Blobs.
+ */
+
+class Lucy::Util::BlobSortEx
+ inherits Lucy::Util::SortExternal {
+
+ Vector *external;
+ uint32_t external_tick;
+ uint32_t mem_consumed;
+
+ inert BlobSortEx*
+ new(uint32_t mem_thresh = 0x1000000, Vector *external = NULL);
+
+ inert BlobSortEx*
+ init(BlobSortEx *self, uint32_t mem_thresh = 0x1000000,
+ Vector *external = NULL);
+
+ void
+ Feed(BlobSortEx *self, decremented Obj *item);
+
+ void
+ Flush(BlobSortEx *self);
+
+ uint32_t
+ Refill(BlobSortEx *self);
+
+ void
+ Clear_Buffer(BlobSortEx *self);
+
+ void
+ Flip(BlobSortEx *self);
+
+ int
+ Compare(BlobSortEx *self, void *va, void *vb);
+
+ incremented Vector*
+ Peek_Cache(BlobSortEx *self);
+
+ incremented nullable Obj*
+ Peek_Last(BlobSortEx *self);
+
+ uint32_t
+ Get_Num_Runs(BlobSortEx *self);
+
+ public void
+ Destroy(BlobSortEx *self);
+}
+
+
http://git-wip-us.apache.org/repos/asf/lucy/blob/2f7df540/perl/t/015-sort_external.t
----------------------------------------------------------------------
diff --git a/perl/t/015-sort_external.t b/perl/t/015-sort_external.t
index b656abf..403dab0 100644
--- a/perl/t/015-sort_external.t
+++ b/perl/t/015-sort_external.t
@@ -23,12 +23,12 @@ use bytes qw();
my ( $sortex, $buffer, @orig, @sort_output );
-$sortex = Lucy::Util::BBSortEx->new( mem_thresh => 4 );
-$sortex->feed( new_bytebuf('c') );
+$sortex = Lucy::Util::BlobSortEx->new( mem_thresh => 4 );
+$sortex->feed( new_blob('c') );
is( $sortex->buffer_count, 1, "feed elem into buffer" );
-$sortex->feed( new_bytebuf('b') );
-$sortex->feed( new_bytebuf('d') );
+$sortex->feed( new_blob('b') );
+$sortex->feed( new_blob('d') );
$sortex->sort_buffer;
SKIP: {
skip( "Restore when porting test to C", 1 );
@@ -36,13 +36,13 @@ SKIP: {
is_deeply( $buffer, [qw( b c d )], "sort buffer" );
}
-$sortex->feed( new_bytebuf('a') );
+$sortex->feed( new_blob('a') );
is( $sortex->buffer_count, 0,
"buffer flushed automatically when mem_thresh crossed" );
#is( $sortex->get_num_runs, 1, "run added" );
-my @bytebufs = map { new_bytebuf($_) } qw( x y z );
-my $run = Lucy::Util::BBSortEx->new( external => \@bytebufs );
+my @blobs = map { new_blob($_) } qw( x y z );
+my $run = Lucy::Util::BlobSortEx->new( external => \@blobs );
$sortex->add_run($run);
$sortex->flip;
@orig = qw( a b c d x y z );
@@ -53,12 +53,12 @@ is_deeply( \@sort_output, \@orig, "Add_Run" );
@orig = ();
@sort_output = ();
-$sortex = Lucy::Util::BBSortEx->new( mem_thresh => 4 );
-$sortex->feed( new_bytebuf('c') );
+$sortex = Lucy::Util::BlobSortEx->new( mem_thresh => 4 );
+$sortex->feed( new_blob('c') );
$sortex->clear_buffer;
is( $sortex->buffer_count, 0, "Clear_Buffer" );
-$sortex->feed( new_bytebuf('b') );
-$sortex->feed( new_bytebuf('a') );
+$sortex->feed( new_blob('b') );
+$sortex->feed( new_blob('a') );
$sortex->flush;
$sortex->flip;
@orig = qw( a b );
@@ -72,9 +72,9 @@ is_deeply( \@sort_output, \@orig,
@orig = ();
@sort_output = ();
-$sortex = Lucy::Util::BBSortEx->new;
+$sortex = Lucy::Util::BlobSortEx->new;
@orig = ( 'a' .. 'z' );
-$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+$sortex->feed( new_blob($_) ) for shuffle(@orig);
$sortex->flip;
while ( defined( my $result = $sortex->fetch ) ) {
push @sort_output, $result;
@@ -83,9 +83,9 @@ is_deeply( \@sort_output, \@orig, "sort letters" );
@orig = ();
@sort_output = ();
-$sortex = Lucy::Util::BBSortEx->new;
+$sortex = Lucy::Util::BlobSortEx->new;
@orig = qw( a a a b c d x x x x x x y y );
-$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+$sortex->feed( new_blob($_) ) for shuffle(@orig);
$sortex->flip;
while ( defined( my $result = $sortex->fetch ) ) {
push @sort_output, $result;
@@ -94,9 +94,9 @@ is_deeply( \@sort_output, \@orig, "sort repeated letters" );
@orig = ();
@sort_output = ();
-$sortex = Lucy::Util::BBSortEx->new;
+$sortex = Lucy::Util::BlobSortEx->new;
@orig = ( '', '', 'a' .. 'z' );
-$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+$sortex->feed( new_blob($_) ) for shuffle(@orig);
$sortex->flip;
while ( defined( my $result = $sortex->fetch ) ) {
push @sort_output, $result;
@@ -105,9 +105,9 @@ is_deeply( \@sort_output, \@orig, "sort letters and empty strings" );
@orig = ();
@sort_output = ();
-$sortex = Lucy::Util::BBSortEx->new( mem_thresh => 30 );
+$sortex = Lucy::Util::BlobSortEx->new( mem_thresh => 30 );
@orig = 'a' .. 'z';
-$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+$sortex->feed( new_blob($_) ) for shuffle(@orig);
$sortex->flip;
while ( defined( my $result = $sortex->fetch ) ) {
push @sort_output, $result;
@@ -116,9 +116,9 @@ is_deeply( \@sort_output, \@orig, "... with an absurdly low mem_thresh" );
@orig = ();
@sort_output = ();
-$sortex = Lucy::Util::BBSortEx->new( mem_thresh => 1 );
+$sortex = Lucy::Util::BlobSortEx->new( mem_thresh => 1 );
@orig = 'a' .. 'z';
-$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+$sortex->feed( new_blob($_) ) for shuffle(@orig);
$sortex->flip;
while ( defined( my $result = $sortex->fetch ) ) {
push @sort_output, $result;
@@ -127,15 +127,15 @@ is_deeply( \@sort_output, \@orig, "... with an even lower mem_thresh" );
@orig = ();
@sort_output = ();
-$sortex = Lucy::Util::BBSortEx->new;
+$sortex = Lucy::Util::BlobSortEx->new;
$sortex->flip;
@sort_output = $sortex->fetch;
is_deeply( \@sort_output, [undef], "Sorting nothing returns undef" );
@sort_output = ();
-$sortex = Lucy::Util::BBSortEx->new( mem_thresh => 5_000 );
+$sortex = Lucy::Util::BlobSortEx->new( mem_thresh => 5_000 );
@orig = map { pack( 'N', $_ ) } ( 0 .. 11_000 );
-$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+$sortex->feed( new_blob($_) ) for shuffle(@orig);
$sortex->flip;
while ( defined( my $item = $sortex->fetch ) ) {
push @sort_output, $item;
@@ -143,7 +143,7 @@ while ( defined( my $item = $sortex->fetch ) ) {
is_deeply( \@sort_output, \@orig, "Sorting packed integers..." );
@sort_output = ();
-$sortex = Lucy::Util::BBSortEx->new( mem_thresh => 15_000 );
+$sortex = Lucy::Util::BlobSortEx->new( mem_thresh => 15_000 );
@orig = ();
for my $iter ( 0 .. 1_000 ) {
my $string = '';
@@ -152,7 +152,7 @@ for my $iter ( 0 .. 1_000 ) {
}
push @orig, $string;
}
-$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+$sortex->feed( new_blob($_) ) for shuffle(@orig);
@orig = sort @orig;
$sortex->flip;
while ( defined( my $item = $sortex->fetch ) ) {
@@ -161,5 +161,5 @@ while ( defined( my $item = $sortex->fetch ) ) {
is_deeply( \@sort_output, \@orig, "Random binary strings of random length" );
@sort_output = ();
-sub new_bytebuf { Clownfish::ByteBuf->new(shift) }
+sub new_blob { Clownfish::Blob->new(shift) }
[4/9] lucy git commit: Switch BlobType fields over to Blob
Posted by nw...@apache.org.
Switch BlobType fields over to Blob
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/4313e9a7
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/4313e9a7
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/4313e9a7
Branch: refs/heads/master
Commit: 4313e9a7b540347de312ba9560d0aaa7f6dcde4d
Parents: 2aa3da2
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Mon May 4 15:55:33 2015 +0200
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Tue May 5 11:23:05 2015 +0200
----------------------------------------------------------------------
c/src/Lucy/Index/DocReader.c | 5 ++---
c/src/Lucy/Index/Inverter.c | 10 ++++------
core/Lucy/Index/DocWriter.c | 5 +++--
core/Lucy/Index/Inverter.c | 2 +-
core/Lucy/Index/SortFieldWriter.c | 11 ++++++-----
perl/xs/Lucy/Index/Inverter.c | 8 ++++----
6 files changed, 20 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/4313e9a7/c/src/Lucy/Index/DocReader.c
----------------------------------------------------------------------
diff --git a/c/src/Lucy/Index/DocReader.c b/c/src/Lucy/Index/DocReader.c
index 0e97ecf..a908680 100644
--- a/c/src/Lucy/Index/DocReader.c
+++ b/c/src/Lucy/Index/DocReader.c
@@ -20,7 +20,7 @@
#define LUCY_USE_SHORT_NAMES
#include "Lucy/Index/DocReader.h"
-#include "Clownfish/ByteBuf.h"
+#include "Clownfish/Blob.h"
#include "Clownfish/String.h"
#include "Clownfish/Err.h"
#include "Clownfish/Hash.h"
@@ -83,8 +83,7 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) {
uint32_t value_len = InStream_Read_C32(dat_in);
char *buf = (char*)MALLOCATE(value_len);
InStream_Read_Bytes(dat_in, buf, value_len);
- value = (Obj*)BB_new_steal_bytes(
- buf, value_len, value_len);
+ value = (Obj*)Blob_new_steal(buf, value_len);
break;
}
case FType_FLOAT32:
http://git-wip-us.apache.org/repos/asf/lucy/blob/4313e9a7/c/src/Lucy/Index/Inverter.c
----------------------------------------------------------------------
diff --git a/c/src/Lucy/Index/Inverter.c b/c/src/Lucy/Index/Inverter.c
index 8393455..23622d8 100644
--- a/c/src/Lucy/Index/Inverter.c
+++ b/c/src/Lucy/Index/Inverter.c
@@ -20,7 +20,7 @@
#define LUCY_USE_SHORT_NAMES
#include "Lucy/Index/Inverter.h"
-#include "Clownfish/ByteBuf.h"
+#include "Clownfish/Blob.h"
#include "Clownfish/String.h"
#include "Clownfish/Err.h"
#include "Clownfish/Hash.h"
@@ -86,11 +86,9 @@ Inverter_Invert_Doc_IMP(Inverter *self, Doc *doc) {
break;
}
case FType_BLOB: {
- ByteBuf *byte_buf
- = (ByteBuf*)CERTIFY(obj, BYTEBUF);
- ViewByteBuf *value
- = (ViewByteBuf*)inventry_ivars->value;
- ViewBB_Assign(value, byte_buf);
+ Blob *blob = (Blob*)CERTIFY(obj, BLOB);
+ DECREF(inventry_ivars->value);
+ inventry_ivars->value = INCREF(blob);
break;
}
case FType_INT32: {
http://git-wip-us.apache.org/repos/asf/lucy/blob/4313e9a7/core/Lucy/Index/DocWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/DocWriter.c b/core/Lucy/Index/DocWriter.c
index 8b70d14..07129a7 100644
--- a/core/Lucy/Index/DocWriter.c
+++ b/core/Lucy/Index/DocWriter.c
@@ -17,6 +17,7 @@
#define C_LUCY_DOCWRITER
#include "Lucy/Util/ToolSet.h"
+#include "Clownfish/Blob.h"
#include "Lucy/Index/DocWriter.h"
#include "Lucy/Document/Doc.h"
#include "Lucy/Index/DocReader.h"
@@ -122,8 +123,8 @@ DocWriter_Add_Inverted_Doc_IMP(DocWriter *self, Inverter *inverter,
break;
}
case FType_BLOB: {
- const char *buf = BB_Get_Buf((ByteBuf*)value);
- size_t size = BB_Get_Size((ByteBuf*)value);
+ const char *buf = Blob_Get_Buf((Blob*)value);
+ size_t size = Blob_Get_Size((Blob*)value);
OutStream_Write_C32(dat_out, size);
OutStream_Write_Bytes(dat_out, buf, size);
break;
http://git-wip-us.apache.org/repos/asf/lucy/blob/4313e9a7/core/Lucy/Index/Inverter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Inverter.c b/core/Lucy/Index/Inverter.c
index dc356b6..23dd0f6 100644
--- a/core/Lucy/Index/Inverter.c
+++ b/core/Lucy/Index/Inverter.c
@@ -218,7 +218,7 @@ InvEntry_init(InverterEntry *self, Schema *schema, String *field,
ivars->value = NULL;
break;
case FType_BLOB:
- ivars->value = (Obj*)ViewBB_new(NULL, 0);
+ ivars->value = NULL;
break;
case FType_INT32:
ivars->value = (Obj*)Int32_new(0);
http://git-wip-us.apache.org/repos/asf/lucy/blob/4313e9a7/core/Lucy/Index/SortFieldWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortFieldWriter.c b/core/Lucy/Index/SortFieldWriter.c
index ff41e73..b40fae7 100644
--- a/core/Lucy/Index/SortFieldWriter.c
+++ b/core/Lucy/Index/SortFieldWriter.c
@@ -35,6 +35,7 @@
#include "Lucy/Store/Folder.h"
#include "Lucy/Store/InStream.h"
#include "Lucy/Store/OutStream.h"
+#include "Clownfish/Blob.h"
#include "Clownfish/Util/Memory.h"
#include "Clownfish/Util/SortUtils.h"
@@ -126,7 +127,7 @@ SortFieldWriter_init(SortFieldWriter *self, Schema *schema,
ivars->var_width = true;
}
else if (ivars->prim_id == FType_BLOB) {
- ivars->mem_per_entry += Class_Get_Obj_Alloc_Size(BYTEBUF);
+ ivars->mem_per_entry += Class_Get_Obj_Alloc_Size(BLOB);
ivars->var_width = true;
}
else {
@@ -180,7 +181,7 @@ SortFieldWriter_Add_IMP(SortFieldWriter *self, int32_t doc_id, Obj *value) {
Counter_Add(counter, size);
}
else if (ivars->prim_id == FType_BLOB) {
- int64_t size = BB_Get_Size((ByteBuf*)value) + 1;
+ int64_t size = Blob_Get_Size((Blob*)value) + 1;
size = SI_increase_to_word_multiple(size);
Counter_Add(counter, size);
}
@@ -266,11 +267,11 @@ S_write_val(Obj *val, int8_t prim_id, OutStream *ix_out, OutStream *dat_out,
break;
}
case FType_BLOB: {
- ByteBuf *byte_buf = (ByteBuf*)val;
+ Blob *blob = (Blob*)val;
int64_t dat_pos = OutStream_Tell(dat_out) - dat_start;
OutStream_Write_I64(ix_out, dat_pos);
- OutStream_Write_Bytes(dat_out, BB_Get_Buf(byte_buf),
- BB_Get_Size(byte_buf));
+ OutStream_Write_Bytes(dat_out, Blob_Get_Buf(blob),
+ Blob_Get_Size(blob));
break;
}
case FType_INT32: {
http://git-wip-us.apache.org/repos/asf/lucy/blob/4313e9a7/perl/xs/Lucy/Index/Inverter.c
----------------------------------------------------------------------
diff --git a/perl/xs/Lucy/Index/Inverter.c b/perl/xs/Lucy/Index/Inverter.c
index 264548e..9f3d6ef 100644
--- a/perl/xs/Lucy/Index/Inverter.c
+++ b/perl/xs/Lucy/Index/Inverter.c
@@ -20,7 +20,7 @@
#include "Lucy/Index/Inverter.h"
#include "Lucy/Document/Doc.h"
#include "Lucy/Index/Segment.h"
-#include "Clownfish/ByteBuf.h"
+#include "Clownfish/Blob.h"
#include "Lucy/Plan/FieldType.h"
#include "Lucy/Plan/BlobType.h"
#include "Lucy/Plan/NumericType.h"
@@ -109,9 +109,9 @@ LUCY_Inverter_Invert_Doc_IMP(lucy_Inverter *self, lucy_Doc *doc) {
case lucy_FType_BLOB: {
STRLEN val_len;
char *val_ptr = SvPV(value_sv, val_len);
- cfish_ViewByteBuf *value
- = (cfish_ViewByteBuf*)entry_ivars->value;
- CFISH_ViewBB_Assign_Bytes(value, val_ptr, val_len);
+ CFISH_DECREF(entry_ivars->value);
+ entry_ivars->value
+ = (cfish_Obj*)cfish_Blob_new_wrap(val_ptr, val_len);
break;
}
case lucy_FType_INT32: {
[2/9] lucy git commit: Switch tv_cache over to Blob
Posted by nw...@apache.org.
Switch tv_cache over to Blob
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/32e5dc4e
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/32e5dc4e
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/32e5dc4e
Branch: refs/heads/master
Commit: 32e5dc4e9f9015dc7d72fb3a884fce38b7a9ef5f
Parents: 9b2f567
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Mon May 4 16:31:02 2015 +0200
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Tue May 5 11:23:05 2015 +0200
----------------------------------------------------------------------
core/Lucy/Index/DocVector.c | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/32e5dc4e/core/Lucy/Index/DocVector.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/DocVector.c b/core/Lucy/Index/DocVector.c
index b9619b7..cc08125 100644
--- a/core/Lucy/Index/DocVector.c
+++ b/core/Lucy/Index/DocVector.c
@@ -33,8 +33,7 @@ S_extract_tv_cache(Blob *field_buf);
// Pull a TermVector object out from compressed positional data.
static TermVector*
-S_extract_tv_from_tv_buf(String *field, String *term_text,
- ByteBuf *tv_buf);
+S_extract_tv_from_tv_buf(String *field, String *term_text, Blob *tv_buf);
DocVector*
DocVec_new() {
@@ -110,7 +109,7 @@ DocVec_Term_Vector_IMP(DocVector *self, String *field,
}
// Get a buf for the term text or bail.
- ByteBuf *tv_buf = (ByteBuf*)Hash_Fetch(field_vector, term_text);
+ Blob *tv_buf = (Blob*)Hash_Fetch(field_vector, term_text);
if (tv_buf == NULL) {
return NULL;
}
@@ -148,7 +147,7 @@ S_extract_tv_cache(Blob *field_buf) {
// Store the $text => $posdata pair in the output hash.
String *text = CB_To_String(text_buf);
- Hash_Store(tv_cache, text, (Obj*)BB_new_bytes(bookmark_ptr, len));
+ Hash_Store(tv_cache, text, (Obj*)Blob_new(bookmark_ptr, len));
DECREF(text);
}
DECREF(text_buf);
@@ -157,11 +156,10 @@ S_extract_tv_cache(Blob *field_buf) {
}
static TermVector*
-S_extract_tv_from_tv_buf(String *field, String *term_text,
- ByteBuf *tv_buf) {
+S_extract_tv_from_tv_buf(String *field, String *term_text, Blob *tv_buf) {
TermVector *retval = NULL;
- const char *posdata = BB_Get_Buf(tv_buf);
- const char *posdata_end = posdata + BB_Get_Size(tv_buf);
+ const char *posdata = Blob_Get_Buf(tv_buf);
+ const char *posdata_end = posdata + Blob_Get_Size(tv_buf);
int32_t *positions = NULL;
int32_t *starts = NULL;
int32_t *ends = NULL;