You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by ma...@apache.org on 2016/04/21 23:13:37 UTC
[01/18] lucy git commit: Duplicate IO C32 as explicitly unsigned CU32.
Repository: lucy
Updated Branches:
refs/heads/master 8b4ad2105 -> e6b61338c
Duplicate IO C32 as explicitly unsigned CU32.
Prepare for the addition of CI32 -- a signed compressed integer.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/8a48ef58
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/8a48ef58
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/8a48ef58
Branch: refs/heads/master
Commit: 8a48ef58e2592336594bebb08a25ea763f83037c
Parents: 4370756
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Mon Apr 18 12:32:54 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Mon Apr 18 15:12:09 2016 -0700
----------------------------------------------------------------------
core/Lucy/Store/InStream.c | 26 ++++++++++++++++++++++++
core/Lucy/Store/InStream.cfh | 17 +++++++++++++---
core/Lucy/Store/OutStream.c | 30 ++++++++++++++++++++++------
core/Lucy/Store/OutStream.cfh | 10 ++++++++++
core/Lucy/Test/Store/TestIOPrimitives.c | 26 ++++++++++++------------
5 files changed, 87 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/8a48ef58/core/Lucy/Store/InStream.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/InStream.c b/core/Lucy/Store/InStream.c
index a6d61d8..09da76f 100644
--- a/core/Lucy/Store/InStream.c
+++ b/core/Lucy/Store/InStream.c
@@ -49,6 +49,12 @@ S_fill(InStream *self, int64_t amount);
static int64_t
S_refill(InStream *self);
+static CFISH_INLINE uint32_t
+SI_read_cu32(InStream *self);
+
+static CFISH_INLINE uint64_t
+SI_read_cu64(InStream *self);
+
InStream*
InStream_open(Obj *file) {
InStream *self = (InStream*)Class_Make_Obj(INSTREAM);
@@ -468,6 +474,16 @@ InStream_Read_F64_IMP(InStream *self) {
uint32_t
InStream_Read_C32_IMP(InStream *self) {
+ return SI_read_cu32(self);
+}
+
+uint32_t
+InStream_Read_CU32_IMP(InStream *self) {
+ return SI_read_cu32(self);
+}
+
+static CFISH_INLINE uint32_t
+SI_read_cu32(InStream *self) {
InStreamIVARS *const ivars = InStream_IVARS(self);
uint32_t retval = 0;
while (1) {
@@ -482,6 +498,16 @@ InStream_Read_C32_IMP(InStream *self) {
uint64_t
InStream_Read_C64_IMP(InStream *self) {
+ return SI_read_cu64(self);
+}
+
+uint64_t
+InStream_Read_CU64_IMP(InStream *self) {
+ return SI_read_cu64(self);
+}
+
+static CFISH_INLINE uint64_t
+SI_read_cu64(InStream *self) {
InStreamIVARS *const ivars = InStream_IVARS(self);
uint64_t retval = 0;
while (1) {
http://git-wip-us.apache.org/repos/asf/lucy/blob/8a48ef58/core/Lucy/Store/InStream.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/InStream.cfh b/core/Lucy/Store/InStream.cfh
index efa7f63..5fd5dab 100644
--- a/core/Lucy/Store/InStream.cfh
+++ b/core/Lucy/Store/InStream.cfh
@@ -176,15 +176,26 @@ class Lucy::Store::InStream inherits Clownfish::Obj {
uint32_t
Read_C32(InStream *self);
+ /** Read in a compressed 32-bit unsigned integer.
+ */
+ uint32_t
+ Read_CU32(InStream *self);
+
/** Read a 64-bit integer, using the same encoding as a C32 but occupying
* as many as 10 bytes.
*/
final uint64_t
Read_C64(InStream *self);
- /** Read the bytes for a C32/C64 into `buf`. Return the number
- * of bytes read. The caller must ensure that sufficient space exists in
- * `buf` (worst case is 10 bytes).
+ /** Read a 64-bit unsigned integer, using the same encoding as a CU32 but
+ * occupying as many as 10 bytes.
+ */
+ final uint64_t
+ Read_CU64(InStream *self);
+
+ /** Read the bytes for a compressed 64-bit integer into `buf`. Return the
+ * number of bytes read. The caller must ensure that sufficient space
+ * exists in `buf` (worst case is 10 bytes).
*/
final int
Read_Raw_C64(InStream *self, char *buf);
http://git-wip-us.apache.org/repos/asf/lucy/blob/8a48ef58/core/Lucy/Store/OutStream.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/OutStream.c b/core/Lucy/Store/OutStream.c
index d0fadd4..8839791 100644
--- a/core/Lucy/Store/OutStream.c
+++ b/core/Lucy/Store/OutStream.c
@@ -33,9 +33,13 @@ static CFISH_INLINE void
SI_write_bytes(OutStream *self, OutStreamIVARS *ivars,
const void *bytes, size_t len);
-// Inlined version of OutStream_Write_C32.
+// Inlined version of OutStream_Write_CU32.
static CFISH_INLINE void
-SI_write_c32(OutStream *self, OutStreamIVARS *ivars, uint32_t value);
+SI_write_cu32(OutStream *self, OutStreamIVARS *ivars, uint32_t value);
+
+// Inlined version of OutStream_Write_CU64.
+static CFISH_INLINE void
+SI_write_cu64(OutStream *self, OutStreamIVARS *ivars, uint64_t value);
// Flush content in the buffer to the FileHandle.
static void
@@ -287,11 +291,16 @@ OutStream_Write_F64_IMP(OutStream *self, double value) {
void
OutStream_Write_C32_IMP(OutStream *self, uint32_t value) {
- SI_write_c32(self, OutStream_IVARS(self), value);
+ SI_write_cu32(self, OutStream_IVARS(self), value);
+}
+
+void
+OutStream_Write_CU32_IMP(OutStream *self, uint32_t value) {
+ SI_write_cu32(self, OutStream_IVARS(self), value);
}
static CFISH_INLINE void
-SI_write_c32(OutStream *self, OutStreamIVARS *ivars, uint32_t value) {
+SI_write_cu32(OutStream *self, OutStreamIVARS *ivars, uint32_t value) {
uint8_t buf[C32_MAX_BYTES];
uint8_t *ptr = buf + sizeof(buf) - 1;
@@ -310,7 +319,16 @@ SI_write_c32(OutStream *self, OutStreamIVARS *ivars, uint32_t value) {
void
OutStream_Write_C64_IMP(OutStream *self, uint64_t value) {
- OutStreamIVARS *const ivars = OutStream_IVARS(self);
+ SI_write_cu64(self, OutStream_IVARS(self), value);
+}
+
+void
+OutStream_Write_CU64_IMP(OutStream *self, uint64_t value) {
+ SI_write_cu64(self, OutStream_IVARS(self), value);
+}
+
+static CFISH_INLINE void
+SI_write_cu64(OutStream *self, OutStreamIVARS *ivars, uint64_t value) {
uint8_t buf[C64_MAX_BYTES];
uint8_t *ptr = buf + sizeof(buf) - 1;
@@ -330,7 +348,7 @@ OutStream_Write_C64_IMP(OutStream *self, uint64_t value) {
void
OutStream_Write_String_IMP(OutStream *self, const char *string, size_t len) {
OutStreamIVARS *const ivars = OutStream_IVARS(self);
- SI_write_c32(self, ivars, (uint32_t)len);
+ SI_write_cu32(self, ivars, (uint32_t)len);
SI_write_bytes(self, ivars, string, len);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/8a48ef58/core/Lucy/Store/OutStream.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/OutStream.cfh b/core/Lucy/Store/OutStream.cfh
index 6bcacac..3726101 100644
--- a/core/Lucy/Store/OutStream.cfh
+++ b/core/Lucy/Store/OutStream.cfh
@@ -118,11 +118,21 @@ class Lucy::Store::OutStream inherits Clownfish::Obj {
final void
Write_C32(OutStream *self, uint32_t value);
+ /** Write an unsigned 32-bit integer using a compressed format.
+ */
+ final void
+ Write_CU32(OutStream *self, uint32_t value);
+
/** Write a 64-bit integer using a compressed format.
*/
final void
Write_C64(OutStream *self, uint64_t value);
+ /** Write an unsigned 64-bit integer using a compressed format.
+ */
+ final void
+ Write_CU64(OutStream *self, uint64_t value);
+
/** Write an IEEE 764 32-bit floating point number in big-endian byte
* order.
*/
http://git-wip-us.apache.org/repos/asf/lucy/blob/8a48ef58/core/Lucy/Test/Store/TestIOPrimitives.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Test/Store/TestIOPrimitives.c b/core/Lucy/Test/Store/TestIOPrimitives.c
index 551f1ac..7409d7e 100644
--- a/core/Lucy/Test/Store/TestIOPrimitives.c
+++ b/core/Lucy/Test/Store/TestIOPrimitives.c
@@ -242,7 +242,7 @@ test_u64(TestBatchRunner *runner) {
}
static void
-test_c32(TestBatchRunner *runner) {
+test_cu32(TestBatchRunner *runner) {
uint64_t *ints = TestUtils_random_u64s(NULL, 1000, 0, UINT32_MAX);
RAMFile *file = RAMFile_new(NULL, false);
OutStream *outstream = OutStream_open((Obj*)file);
@@ -256,21 +256,21 @@ test_c32(TestBatchRunner *runner) {
ints[3] = UINT32_MAX - 1;
for (i = 0; i < 1000; i++) {
- OutStream_Write_C32(outstream, (uint32_t)ints[i]);
+ OutStream_Write_CU32(outstream, (uint32_t)ints[i]);
}
OutStream_Close(outstream);
instream = InStream_open((Obj*)file);
for (i = 0; i < 1000; i++) {
- uint32_t got = InStream_Read_C32(instream);
+ uint32_t got = InStream_Read_CU32(instream);
if (got != ints[i]) {
- FAIL(runner, "c32 round trip failed: %lu, %lu", (unsigned long)got,
+ FAIL(runner, "cu32 round trip failed: %lu, %lu", (unsigned long)got,
(unsigned long)ints[i]);
break;
}
}
if (i == 1000) {
- PASS(runner, "c32 round trip");
+ PASS(runner, "cu32 round trip");
}
DECREF(instream);
@@ -280,7 +280,7 @@ test_c32(TestBatchRunner *runner) {
}
static void
-test_c64(TestBatchRunner *runner) {
+test_cu64(TestBatchRunner *runner) {
uint64_t *ints = TestUtils_random_u64s(NULL, 1000, 0, UINT64_MAX);
RAMFile *file = RAMFile_new(NULL, false);
RAMFile *raw_file = RAMFile_new(NULL, false);
@@ -297,23 +297,23 @@ test_c64(TestBatchRunner *runner) {
ints[3] = UINT64_MAX - 1;
for (i = 0; i < 1000; i++) {
- OutStream_Write_C64(outstream, ints[i]);
- OutStream_Write_C64(raw_outstream, ints[i]);
+ OutStream_Write_CU64(outstream, ints[i]);
+ OutStream_Write_CU64(raw_outstream, ints[i]);
}
OutStream_Close(outstream);
OutStream_Close(raw_outstream);
instream = InStream_open((Obj*)file);
for (i = 0; i < 1000; i++) {
- uint64_t got = InStream_Read_C64(instream);
+ uint64_t got = InStream_Read_CU64(instream);
if (got != ints[i]) {
- FAIL(runner, "c64 round trip failed: %" PRIu64 ", %" PRIu64,
+ FAIL(runner, "cu64 round trip failed: %" PRIu64 ", %" PRIu64,
got, ints[i]);
break;
}
}
if (i == 1000) {
- PASS(runner, "c64 round trip");
+ PASS(runner, "cu64 round trip");
}
raw_instream = InStream_open((Obj*)raw_file);
@@ -429,8 +429,8 @@ TestIOPrimitives_Run_IMP(TestIOPrimitives *self, TestBatchRunner *runner) {
test_u32(runner);
test_i64(runner);
test_u64(runner);
- test_c32(runner);
- test_c64(runner);
+ test_cu32(runner);
+ test_cu64(runner);
test_f32(runner);
test_f64(runner);
}
[14/18] lucy git commit: Switch C64 to CI64 for file positions.
Posted by ma...@apache.org.
Switch C64 to CI64 for file positions.
Change Read/Write C64 to CI64 for file positions, all of which are
represented as signed 64-bit integers.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/e5ba8a19
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/e5ba8a19
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/e5ba8a19
Branch: refs/heads/master
Commit: e5ba8a19dd8e04039e7de5479b852150b7705b46
Parents: 5e4dcd2
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Wed Apr 20 18:34:30 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Wed Apr 20 19:26:48 2016 -0700
----------------------------------------------------------------------
core/Lucy/Index/LexIndex.c | 4 ++--
core/Lucy/Index/LexiconWriter.c | 2 +-
core/Lucy/Index/Posting/MatchPosting.c | 16 ++++++++--------
core/Lucy/Index/SkipStepper.c | 4 ++--
4 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/e5ba8a19/core/Lucy/Index/LexIndex.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/LexIndex.c b/core/Lucy/Index/LexIndex.c
index 2c6586b..8a406a3 100644
--- a/core/Lucy/Index/LexIndex.c
+++ b/core/Lucy/Index/LexIndex.c
@@ -131,12 +131,12 @@ S_read_entry(LexIndex *self) {
TermStepper_Read_Key_Frame(ivars->term_stepper, ix_in);
int32_t doc_freq = InStream_Read_CI32(ix_in);
TInfo_Set_Doc_Freq(tinfo, doc_freq);
- TInfo_Set_Post_FilePos(tinfo, InStream_Read_C64(ix_in));
+ TInfo_Set_Post_FilePos(tinfo, InStream_Read_CI64(ix_in));
int64_t skip_filepos = doc_freq >= ivars->skip_interval
? InStream_Read_CI64(ix_in)
: 0;
TInfo_Set_Skip_FilePos(tinfo, skip_filepos);
- TInfo_Set_Lex_FilePos(tinfo, InStream_Read_C64(ix_in));
+ TInfo_Set_Lex_FilePos(tinfo, InStream_Read_CI64(ix_in));
}
void
http://git-wip-us.apache.org/repos/asf/lucy/blob/e5ba8a19/core/Lucy/Index/LexiconWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/LexiconWriter.c b/core/Lucy/Index/LexiconWriter.c
index 1ef32f1..7c996d5 100644
--- a/core/Lucy/Index/LexiconWriter.c
+++ b/core/Lucy/Index/LexiconWriter.c
@@ -100,7 +100,7 @@ S_add_last_term_to_ix(LexiconWriter *self) {
ivars->ix_out, TermStepper_Get_Value(ivars->term_stepper));
TermStepper_Write_Key_Frame(ivars->tinfo_stepper,
ivars->ix_out, TermStepper_Get_Value(ivars->tinfo_stepper));
- OutStream_Write_C64(ivars->ix_out, OutStream_Tell(ivars->dat_out));
+ OutStream_Write_CI64(ivars->ix_out, OutStream_Tell(ivars->dat_out));
ivars->ix_count++;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/e5ba8a19/core/Lucy/Index/Posting/MatchPosting.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Posting/MatchPosting.c b/core/Lucy/Index/Posting/MatchPosting.c
index ae4e1f2..c7348fa 100644
--- a/core/Lucy/Index/Posting/MatchPosting.c
+++ b/core/Lucy/Index/Posting/MatchPosting.c
@@ -275,11 +275,11 @@ MatchTInfoStepper_Write_Key_Frame_IMP(MatchTermInfoStepper *self,
OutStream_Write_C32(outstream, doc_freq);
// Write postings file pointer.
- OutStream_Write_C64(outstream, tinfo_ivars->post_filepos);
+ OutStream_Write_CI64(outstream, tinfo_ivars->post_filepos);
// Write skip file pointer (maybe).
if (doc_freq >= ivars->skip_interval) {
- OutStream_Write_C64(outstream, tinfo_ivars->skip_filepos);
+ OutStream_Write_CI64(outstream, tinfo_ivars->skip_filepos);
}
TInfo_Mimic((TermInfo*)ivars->value, (Obj*)tinfo);
@@ -299,11 +299,11 @@ MatchTInfoStepper_Write_Delta_IMP(MatchTermInfoStepper *self,
OutStream_Write_C32(outstream, doc_freq);
// Write postings file pointer delta.
- OutStream_Write_C64(outstream, post_delta);
+ OutStream_Write_CI64(outstream, post_delta);
// Write skip file pointer (maybe).
if (doc_freq >= ivars->skip_interval) {
- OutStream_Write_C64(outstream, TInfo_IVARS(tinfo)->skip_filepos);
+ OutStream_Write_CI64(outstream, TInfo_IVARS(tinfo)->skip_filepos);
}
TInfo_Mimic((TermInfo*)ivars->value, (Obj*)tinfo);
@@ -319,11 +319,11 @@ MatchTInfoStepper_Read_Key_Frame_IMP(MatchTermInfoStepper *self,
tinfo_ivars->doc_freq = InStream_Read_C32(instream);
// Read postings file pointer.
- tinfo_ivars->post_filepos = InStream_Read_C64(instream);
+ tinfo_ivars->post_filepos = InStream_Read_CI64(instream);
// Maybe read skip pointer.
if (tinfo_ivars->doc_freq >= ivars->skip_interval) {
- tinfo_ivars->skip_filepos = InStream_Read_C64(instream);
+ tinfo_ivars->skip_filepos = InStream_Read_CI64(instream);
}
else {
tinfo_ivars->skip_filepos = 0;
@@ -339,11 +339,11 @@ MatchTInfoStepper_Read_Delta_IMP(MatchTermInfoStepper *self, InStream *instream)
tinfo_ivars->doc_freq = InStream_Read_C32(instream);
// Adjust postings file pointer.
- tinfo_ivars->post_filepos += InStream_Read_C64(instream);
+ tinfo_ivars->post_filepos += InStream_Read_CI64(instream);
// Maybe read skip pointer.
if (tinfo_ivars->doc_freq >= ivars->skip_interval) {
- tinfo_ivars->skip_filepos = InStream_Read_C64(instream);
+ tinfo_ivars->skip_filepos = InStream_Read_CI64(instream);
}
else {
tinfo_ivars->skip_filepos = 0;
http://git-wip-us.apache.org/repos/asf/lucy/blob/e5ba8a19/core/Lucy/Index/SkipStepper.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SkipStepper.c b/core/Lucy/Index/SkipStepper.c
index 9f22dc9..634e2d8 100644
--- a/core/Lucy/Index/SkipStepper.c
+++ b/core/Lucy/Index/SkipStepper.c
@@ -48,7 +48,7 @@ void
SkipStepper_Read_Record_IMP(SkipStepper *self, InStream *instream) {
SkipStepperIVARS *const ivars = SkipStepper_IVARS(self);
ivars->doc_id += InStream_Read_C32(instream);
- ivars->filepos += InStream_Read_C64(instream);
+ ivars->filepos += InStream_Read_CI64(instream);
}
String*
@@ -69,7 +69,7 @@ SkipStepper_Write_Record_IMP(SkipStepper *self, OutStream *outstream,
OutStream_Write_C32(outstream, delta_doc_id);
// Write delta file pointer.
- OutStream_Write_C64(outstream, delta_filepos);
+ OutStream_Write_CI64(outstream, delta_filepos);
}
[08/18] lucy git commit: Switch Go bindings from C32/C64 to CI32/etc.
Posted by ma...@apache.org.
Switch Go bindings from C32/C64 to CI32/etc.
Change IO methods invoked to use explicitly signed/unsigned versions
rather than casting.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/795626b6
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/795626b6
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/795626b6
Branch: refs/heads/master
Commit: 795626b6c4c6c98cd6f6ac0d0a2899e4b6e09e33
Parents: cbe964d
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Tue Apr 19 19:19:28 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Tue Apr 19 19:20:27 2016 -0700
----------------------------------------------------------------------
go/build.go | 8 ++++++
go/lucy/lucy.go | 16 +++++------
go/lucy/store.go | 66 +++++++++++++++++++++++++++++++++++++++++++++-
go/lucy/store_test.go | 28 ++++++++++++++------
4 files changed, 101 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/795626b6/go/build.go
----------------------------------------------------------------------
diff --git a/go/build.go b/go/build.go
index 8353d66..ecf36f4 100644
--- a/go/build.go
+++ b/go/build.go
@@ -388,7 +388,11 @@ func specClasses(parcel *cfc.Parcel) {
inStreamBinding.SpecMethod("Read_U32", "ReadU32() (uint32, error)")
inStreamBinding.SpecMethod("Read_U64", "ReadU64() (uint64, error)")
inStreamBinding.SpecMethod("Read_C32", "ReadC32() (uint32, error)")
+ inStreamBinding.SpecMethod("Read_CI32", "ReadCI32() (int32, error)")
+ inStreamBinding.SpecMethod("Read_CU32", "ReadCU32() (uint32, error)")
inStreamBinding.SpecMethod("Read_C64", "ReadC64() (uint64, error)")
+ inStreamBinding.SpecMethod("Read_CI64", "ReadCI64() (int64, error)")
+ inStreamBinding.SpecMethod("Read_CU64", "ReadCU64() (uint64, error)")
inStreamBinding.SpecMethod("Read_F32", "ReadF32() (float32, error)")
inStreamBinding.SpecMethod("Read_F64", "ReadF64() (float64, error)")
inStreamBinding.Register()
@@ -406,7 +410,11 @@ func specClasses(parcel *cfc.Parcel) {
outStreamBinding.SpecMethod("Write_U32", "WriteU32(uint32) error")
outStreamBinding.SpecMethod("Write_U64", "WriteU64(uint64) error")
outStreamBinding.SpecMethod("Write_C32", "WriteC32(uint32) error")
+ outStreamBinding.SpecMethod("Write_CI32", "WriteCI32(int32) error")
+ outStreamBinding.SpecMethod("Write_CU32", "WriteCU32(uint32) error")
outStreamBinding.SpecMethod("Write_C64", "WriteC64(uint64) error")
+ outStreamBinding.SpecMethod("Write_CI64", "WriteCI64(int64) error")
+ outStreamBinding.SpecMethod("Write_CU64", "WriteCU64(uint64) error")
outStreamBinding.SpecMethod("Write_F32", "WriteF32(float32) error")
outStreamBinding.SpecMethod("Write_F64", "WriteF64(float64) error")
outStreamBinding.SpecMethod("Absorb", "Absorb(InStream) error")
http://git-wip-us.apache.org/repos/asf/lucy/blob/795626b6/go/lucy/lucy.go
----------------------------------------------------------------------
diff --git a/go/lucy/lucy.go b/go/lucy/lucy.go
index 138b599..4a4fc71 100644
--- a/go/lucy/lucy.go
+++ b/go/lucy/lucy.go
@@ -297,7 +297,7 @@ func GOLUCY_Doc_Serialize(d *C.lucy_Doc, outstream *C.lucy_OutStream) {
hash := clownfish.GoToClownfish(fields, unsafe.Pointer(C.CFISH_HASH), false)
defer C.cfish_decref(hash)
C.lucy_Freezer_serialize_hash((*C.cfish_Hash)(hash), outstream)
- C.LUCY_OutStream_Write_C32(outstream, C.uint32_t(ivars.doc_id))
+ C.LUCY_OutStream_Write_CI32(outstream, ivars.doc_id)
}
//export GOLUCY_Doc_Deserialize
@@ -308,7 +308,7 @@ func GOLUCY_Doc_Deserialize(d *C.lucy_Doc, instream *C.lucy_InStream) *C.lucy_Do
fields := clownfish.ToGo(hash)
fieldsID := registry.store(fields)
ivars.fields = unsafe.Pointer(fieldsID)
- ivars.doc_id = C.int32_t(C.LUCY_InStream_Read_C32(instream))
+ ivars.doc_id = C.LUCY_InStream_Read_CI32(instream)
return d
}
@@ -473,12 +473,12 @@ func doReadDocData(ddrC *C.lucy_DefaultDocReader, docID int32, doc interface{})
C.LUCY_InStream_Seek(ixInstream, C.int64_t(docID*8))
start := C.LUCY_InStream_Read_U64(ixInstream)
C.LUCY_InStream_Seek(datInstream, C.int64_t(start))
- numFields := uint32(C.LUCY_InStream_Read_C32(datInstream))
+ numFields := uint32(C.LUCY_InStream_Read_CU32(datInstream))
// Decode stored data and build up the doc field by field.
for i := uint32(0); i < numFields; i++ {
// Read field name.
- fieldNameLen := C.size_t(C.LUCY_InStream_Read_C32(datInstream))
+ fieldNameLen := C.size_t(C.LUCY_InStream_Read_CU32(datInstream))
if fieldNameLen > fieldNameCap {
fieldNameCap = fieldNameLen
fieldName = ((*C.char)(C.realloc(unsafe.Pointer(fieldName), fieldNameCap+1)))
@@ -497,7 +497,7 @@ func doReadDocData(ddrC *C.lucy_DefaultDocReader, docID int32, doc interface{})
// Read the field value.
switch C.LUCY_FType_Primitive_ID(fieldType) & C.lucy_FType_PRIMITIVE_ID_MASK {
case C.lucy_FType_TEXT:
- valueLen := C.size_t(C.LUCY_InStream_Read_C32(datInstream))
+ valueLen := C.size_t(C.LUCY_InStream_Read_CU32(datInstream))
buf := ((*C.char)(C.malloc(valueLen + 1)))
C.LUCY_InStream_Read_Bytes(datInstream, buf, valueLen)
val := C.GoStringN(buf, C.int(valueLen))
@@ -506,7 +506,7 @@ func doReadDocData(ddrC *C.lucy_DefaultDocReader, docID int32, doc interface{})
return err
}
case C.lucy_FType_BLOB:
- valueLen := C.size_t(C.LUCY_InStream_Read_C32(datInstream))
+ valueLen := C.size_t(C.LUCY_InStream_Read_CU32(datInstream))
buf := ((*C.char)(C.malloc(valueLen)))
C.LUCY_InStream_Read_Bytes(datInstream, buf, valueLen)
val := C.GoBytes(unsafe.Pointer(buf), C.int(valueLen))
@@ -525,12 +525,12 @@ func doReadDocData(ddrC *C.lucy_DefaultDocReader, docID int32, doc interface{})
return err
}
case C.lucy_FType_INT32:
- err := setField(fields, fieldNameGo, int32(C.LUCY_InStream_Read_C32(datInstream)))
+ err := setField(fields, fieldNameGo, int32(C.LUCY_InStream_Read_CI32(datInstream)))
if err != nil {
return err
}
case C.lucy_FType_INT64:
- err := setField(fields, fieldNameGo, int64(C.LUCY_InStream_Read_C64(datInstream)))
+ err := setField(fields, fieldNameGo, int64(C.LUCY_InStream_Read_CI64(datInstream)))
if err != nil {
return err
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/795626b6/go/lucy/store.go
----------------------------------------------------------------------
diff --git a/go/lucy/store.go b/go/lucy/store.go
index 7af8229..0b11a2f 100644
--- a/go/lucy/store.go
+++ b/go/lucy/store.go
@@ -95,7 +95,7 @@ func (in *InStreamIMP) ReadString() (string, error) {
var retval string
err := clownfish.TrapErr(func() {
self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in"))
- size := C.size_t(C.LUCY_InStream_Read_C32(self))
+ size := C.size_t(C.LUCY_InStream_Read_CU32(self))
buf := (*C.char)(C.malloc(size))
defer C.free(unsafe.Pointer(buf))
C.LUCY_InStream_Read_Bytes(self, buf, size)
@@ -180,6 +180,24 @@ func (in *InStreamIMP) ReadC32() (uint32, error) {
return retval, err
}
+func (in *InStreamIMP) ReadCI32() (int32, error) {
+ var retval int32
+ err := clownfish.TrapErr(func() {
+ self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in"))
+ retval = int32(C.LUCY_InStream_Read_CI32(self))
+ })
+ return retval, err
+}
+
+func (in *InStreamIMP) ReadCU32() (uint32, error) {
+ var retval uint32
+ err := clownfish.TrapErr(func() {
+ self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in"))
+ retval = uint32(C.LUCY_InStream_Read_CU32(self))
+ })
+ return retval, err
+}
+
func (in *InStreamIMP) ReadC64() (uint64, error) {
var retval uint64
err := clownfish.TrapErr(func() {
@@ -189,6 +207,24 @@ func (in *InStreamIMP) ReadC64() (uint64, error) {
return retval, err
}
+func (in *InStreamIMP) ReadCI64() (int64, error) {
+ var retval int64
+ err := clownfish.TrapErr(func() {
+ self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in"))
+ retval = int64(C.LUCY_InStream_Read_CI64(self))
+ })
+ return retval, err
+}
+
+func (in *InStreamIMP) ReadCU64() (uint64, error) {
+ var retval uint64
+ err := clownfish.TrapErr(func() {
+ self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in"))
+ retval = uint64(C.LUCY_InStream_Read_CU64(self))
+ })
+ return retval, err
+}
+
func (in *InStreamIMP) ReadF32() (float32, error) {
var retval float32
err := clownfish.TrapErr(func() {
@@ -314,6 +350,20 @@ func (out *OutStreamIMP) WriteC32(value uint32) error {
})
}
+func (out *OutStreamIMP) WriteCI32(value int32) error {
+ return clownfish.TrapErr(func() {
+ self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out"))
+ C.LUCY_OutStream_Write_CI32(self, C.int32_t(value))
+ })
+}
+
+func (out *OutStreamIMP) WriteCU32(value uint32) error {
+ return clownfish.TrapErr(func() {
+ self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out"))
+ C.LUCY_OutStream_Write_CU32(self, C.uint32_t(value))
+ })
+}
+
func (out *OutStreamIMP) WriteC64(value uint64) error {
return clownfish.TrapErr(func() {
self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out"))
@@ -321,6 +371,20 @@ func (out *OutStreamIMP) WriteC64(value uint64) error {
})
}
+func (out *OutStreamIMP) WriteCI64(value int64) error {
+ return clownfish.TrapErr(func() {
+ self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out"))
+ C.LUCY_OutStream_Write_CI64(self, C.int64_t(value))
+ })
+}
+
+func (out *OutStreamIMP) WriteCU64(value uint64) error {
+ return clownfish.TrapErr(func() {
+ self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out"))
+ C.LUCY_OutStream_Write_CU64(self, C.uint64_t(value))
+ })
+}
+
func (out *OutStreamIMP) WriteF32(value float32) error {
return clownfish.TrapErr(func() {
self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out"))
http://git-wip-us.apache.org/repos/asf/lucy/blob/795626b6/go/lucy/store_test.go
----------------------------------------------------------------------
diff --git a/go/lucy/store_test.go b/go/lucy/store_test.go
index c422525..d87d539 100644
--- a/go/lucy/store_test.go
+++ b/go/lucy/store_test.go
@@ -215,11 +215,17 @@ func TestIOStreamReadWrite(t *testing.T) {
if err = outStream.WriteU64(42); err != nil {
t.Errorf("WriteU64: %s", err)
}
- if err = outStream.WriteC32(42); err != nil {
- t.Errorf("WriteC32: %s", err)
+ if err = outStream.WriteCI32(42); err != nil {
+ t.Errorf("WriteCI32: %s", err)
}
- if err = outStream.WriteC64(42); err != nil {
- t.Errorf("WriteC64: %s", err)
+ if err = outStream.WriteCU32(42); err != nil {
+ t.Errorf("WriteCU32: %s", err)
+ }
+ if err = outStream.WriteCI64(42); err != nil {
+ t.Errorf("WriteCI64: %s", err)
+ }
+ if err = outStream.WriteCU64(42); err != nil {
+ t.Errorf("WriteCU64: %s", err)
}
if err = outStream.WriteF32(1.5); err != nil {
t.Errorf("WriteF32: %s", err)
@@ -265,11 +271,17 @@ func TestIOStreamReadWrite(t *testing.T) {
if got, err := inStream.ReadU64(); got != 42 || err != nil {
t.Errorf("ReadU64: %d, %s", got, err)
}
- if got, err := inStream.ReadC32(); got != 42 || err != nil {
- t.Errorf("ReadC32: %d, %s", got, err)
+ if got, err := inStream.ReadCI32(); got != 42 || err != nil {
+ t.Errorf("ReadCI32: %d, %s", got, err)
+ }
+ if got, err := inStream.ReadCU32(); got != 42 || err != nil {
+ t.Errorf("ReadCU32: %d, %s", got, err)
+ }
+ if got, err := inStream.ReadCI64(); got != 42 || err != nil {
+ t.Errorf("ReadCI64: %d, %s", got, err)
}
- if got, err := inStream.ReadC64(); got != 42 || err != nil {
- t.Errorf("ReadC64: %d, %s", got, err)
+ if got, err := inStream.ReadCU64(); got != 42 || err != nil {
+ t.Errorf("ReadCU64: %d, %s", got, err)
}
if got, err := inStream.ReadF32(); got != 1.5 || err != nil {
t.Errorf("ReadF32: %d, %s", got, err)
[15/18] lucy git commit: Change size_t C32/C64 read/writes.
Posted by ma...@apache.org.
Change size_t C32/C64 read/writes.
Adapt Read_C32/Write_C32/Read_C64/Write_C64 to Read_CU32/etc for
variables representing memory sizes: String, Blob, Vector and Hash sizes
in particular.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/5e4dcd2e
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/5e4dcd2e
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/5e4dcd2e
Branch: refs/heads/master
Commit: 5e4dcd2e31a80ed20e86ca931bbbc565f6781820
Parents: 17a3d60
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Wed Apr 20 18:16:06 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Wed Apr 20 19:26:48 2016 -0700
----------------------------------------------------------------------
c/src/Lucy/Index/DocReader.c | 6 +++---
core/Lucy/Index/DocWriter.c | 12 ++++++++++--
core/Lucy/Index/TermVector.c | 4 ++--
core/Lucy/Plan/TextType.c | 3 ++-
core/Lucy/Search/SortSpec.c | 3 ++-
core/Lucy/Util/Freezer.c | 22 +++++++++++++++-------
6 files changed, 34 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/c/src/Lucy/Index/DocReader.c
----------------------------------------------------------------------
diff --git a/c/src/Lucy/Index/DocReader.c b/c/src/Lucy/Index/DocReader.c
index 3610dfc..16e184e 100644
--- a/c/src/Lucy/Index/DocReader.c
+++ b/c/src/Lucy/Index/DocReader.c
@@ -40,7 +40,7 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) {
Hash *const fields = Hash_new(1);
int64_t start;
uint32_t num_fields;
- uint32_t field_name_cap = 31;
+ size_t field_name_cap = 31;
char *field_name = (char*)MALLOCATE(field_name_cap + 1);
// Get data file pointer from index, read number of fields.
@@ -51,12 +51,12 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) {
// Decode stored data and build up the doc field by field.
while (num_fields--) {
- uint32_t field_name_len;
+ size_t field_name_len;
Obj *value;
FieldType *type;
// Read field name.
- field_name_len = InStream_Read_C32(dat_in);
+ field_name_len = InStream_Read_CU32(dat_in);
if (field_name_len > field_name_cap) {
field_name_cap = field_name_len;
field_name = (char*)REALLOCATE(field_name,
http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Index/DocWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/DocWriter.c b/core/Lucy/Index/DocWriter.c
index 818f45b..ec583cc 100644
--- a/core/Lucy/Index/DocWriter.c
+++ b/core/Lucy/Index/DocWriter.c
@@ -119,14 +119,22 @@ DocWriter_Add_Inverted_Doc_IMP(DocWriter *self, Inverter *inverter,
case FType_TEXT: {
const char *buf = Str_Get_Ptr8((String*)value);
size_t size = Str_Get_Size((String*)value);
- OutStream_Write_C32(dat_out, size);
+ if (size > INT32_MAX) {
+ THROW(ERR, "Field %o over 2GB: %u64", field,
+ (uint64_t)size);
+ }
+ OutStream_Write_CU32(dat_out, (uint32_t)size);
OutStream_Write_Bytes(dat_out, buf, size);
break;
}
case FType_BLOB: {
const char *buf = Blob_Get_Buf((Blob*)value);
size_t size = Blob_Get_Size((Blob*)value);
- OutStream_Write_C32(dat_out, size);
+ if (size > INT32_MAX) {
+ THROW(ERR, "Field %o over 2GB: %u64", field,
+ (uint64_t)size);
+ }
+ OutStream_Write_CU32(dat_out, (uint32_t)size);
OutStream_Write_Bytes(dat_out, buf, size);
break;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Index/TermVector.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/TermVector.c b/core/Lucy/Index/TermVector.c
index b1a3391..9c3a6f9 100644
--- a/core/Lucy/Index/TermVector.c
+++ b/core/Lucy/Index/TermVector.c
@@ -90,7 +90,7 @@ TV_Serialize_IMP(TermVector *self, OutStream *target) {
Freezer_serialize_string(ivars->field, target);
Freezer_serialize_string(ivars->text, target);
- OutStream_Write_C64(target, ivars->num_pos);
+ OutStream_Write_CU64(target, ivars->num_pos);
for (size_t i = 0; i < ivars->num_pos; i++) {
OutStream_Write_C32(target, posits[i]);
@@ -103,7 +103,7 @@ TermVector*
TV_Deserialize_IMP(TermVector *self, InStream *instream) {
String *field = Freezer_read_string(instream);
String *text = Freezer_read_string(instream);
- size_t num_pos = InStream_Read_C64(instream);
+ size_t num_pos = InStream_Read_CU64(instream);
// Read positional data.
int32_t *posits = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Plan/TextType.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Plan/TextType.c b/core/Lucy/Plan/TextType.c
index 4b84fa8..67f64a3 100644
--- a/core/Lucy/Plan/TextType.c
+++ b/core/Lucy/Plan/TextType.c
@@ -101,7 +101,8 @@ TextTermStepper_Write_Key_Frame_IMP(TextTermStepper *self,
String *string = (String*)CERTIFY(value, STRING);
const char *buf = Str_Get_Ptr8(string);
size_t size = Str_Get_Size(string);
- OutStream_Write_C32(outstream, size);
+ // Skip size check because we're deep in the internals.
+ OutStream_Write_CU32(outstream, (uint32_t)size);
OutStream_Write_Bytes(outstream, buf, size);
S_set_value(self, value);
http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Search/SortSpec.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Search/SortSpec.c b/core/Lucy/Search/SortSpec.c
index 9994bb4..21a6370 100644
--- a/core/Lucy/Search/SortSpec.c
+++ b/core/Lucy/Search/SortSpec.c
@@ -75,8 +75,9 @@ SortSpec_Get_Rules_IMP(SortSpec *self) {
void
SortSpec_Serialize_IMP(SortSpec *self, OutStream *target) {
SortSpecIVARS *const ivars = SortSpec_IVARS(self);
+ // Skip size check.
uint32_t num_rules = (uint32_t)Vec_Get_Size(ivars->rules);
- OutStream_Write_C32(target, num_rules);
+ OutStream_Write_CU32(target, num_rules);
for (uint32_t i = 0; i < num_rules; i++) {
SortRule *rule = (SortRule*)Vec_Fetch(ivars->rules, i);
SortRule_Serialize(rule, target);
http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Util/Freezer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/Freezer.c b/core/Lucy/Util/Freezer.c
index 20e5e33..90a63c3 100644
--- a/core/Lucy/Util/Freezer.c
+++ b/core/Lucy/Util/Freezer.c
@@ -180,13 +180,16 @@ void
Freezer_serialize_string(String *string, OutStream *outstream) {
size_t size = Str_Get_Size(string);
const char *buf = Str_Get_Ptr8(string);
- OutStream_Write_C64(outstream, size);
+ if (size > INT32_MAX) {
+ THROW(ERR, "Can't serialize string above 2GB: %u64", (uint64_t)size);
+ }
+ OutStream_Write_CU64(outstream, size);
OutStream_Write_Bytes(outstream, buf, size);
}
String*
Freezer_deserialize_string(String *string, InStream *instream) {
- size_t size = InStream_Read_C32(instream);
+ size_t size = InStream_Read_CU32(instream);
if (size == SIZE_MAX) {
THROW(ERR, "Can't deserialize SIZE_MAX bytes");
}
@@ -208,13 +211,16 @@ Freezer_read_string(InStream *instream) {
void
Freezer_serialize_blob(Blob *blob, OutStream *outstream) {
size_t size = Blob_Get_Size(blob);
- OutStream_Write_C32(outstream, size);
+ if (size > INT32_MAX) {
+ THROW(ERR, "Can't serialize blob above 2GB: %u64", (uint64_t)size);
+ }
+ OutStream_Write_CU64(outstream, size);
OutStream_Write_Bytes(outstream, Blob_Get_Buf(blob), size);
}
Blob*
Freezer_deserialize_blob(Blob *blob, InStream *instream) {
- size_t size = InStream_Read_C32(instream);
+ size_t size = InStream_Read_CU32(instream);
char *buf = (char*)MALLOCATE(size);
InStream_Read_Bytes(instream, buf, size);
return Blob_init_steal(blob, buf, size);
@@ -229,8 +235,9 @@ Freezer_read_blob(InStream *instream) {
void
Freezer_serialize_varray(Vector *array, OutStream *outstream) {
uint32_t last_valid_tick = 0;
+ // Skip size check.
uint32_t size = (uint32_t)Vec_Get_Size(array);
- OutStream_Write_C32(outstream, size);
+ OutStream_Write_CU32(outstream, size);
for (uint32_t i = 0; i < size; i++) {
Obj *elem = Vec_Fetch(array, i);
if (elem) {
@@ -266,8 +273,9 @@ Freezer_read_varray(InStream *instream) {
void
Freezer_serialize_hash(Hash *hash, OutStream *outstream) {
- uint32_t hash_size = Hash_Get_Size(hash);
- OutStream_Write_C32(outstream, hash_size);
+ // Skip size check.
+ uint32_t hash_size = (uint32_t)Hash_Get_Size(hash);
+ OutStream_Write_CU32(outstream, hash_size);
HashIterator *iter = HashIter_new(hash);
while (HashIter_Next(iter)) {
[13/18] lucy git commit: Misc C32/C64 transition.
Posted by ma...@apache.org.
Misc C32/C64 transition.
Transition all remaining Read/Write C32/C64 to Read_CI64/etc.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/febd2ff3
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/febd2ff3
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/febd2ff3
Branch: refs/heads/master
Commit: febd2ff3cb76b460e0778e6d732830cf1f860e50
Parents: 656a773
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Wed Apr 20 19:01:15 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Wed Apr 20 19:26:48 2016 -0700
----------------------------------------------------------------------
c/src/Lucy/Index/DocReader.c | 2 +-
core/Lucy/Index/DocWriter.c | 6 +++---
core/Lucy/Index/HighlightWriter.c | 2 +-
core/Lucy/Index/Posting/MatchPosting.c | 6 +++---
core/Lucy/Plan/TextType.c | 2 +-
core/Lucy/Search/SortRule.c | 8 ++++----
core/Lucy/Search/TopDocs.c | 4 ++--
core/Lucy/Util/Freezer.c | 6 +++---
core/LucyX/Search/ProximityQuery.c | 6 +++---
9 files changed, 21 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/febd2ff3/c/src/Lucy/Index/DocReader.c
----------------------------------------------------------------------
diff --git a/c/src/Lucy/Index/DocReader.c b/c/src/Lucy/Index/DocReader.c
index 16e184e..97c8df0 100644
--- a/c/src/Lucy/Index/DocReader.c
+++ b/c/src/Lucy/Index/DocReader.c
@@ -47,7 +47,7 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) {
InStream_Seek(ix_in, (int64_t)doc_id * 8);
start = InStream_Read_U64(ix_in);
InStream_Seek(dat_in, start);
- num_fields = InStream_Read_C32(dat_in);
+ num_fields = InStream_Read_CU32(dat_in);
// Decode stored data and build up the doc field by field.
while (num_fields--) {
http://git-wip-us.apache.org/repos/asf/lucy/blob/febd2ff3/core/Lucy/Index/DocWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/DocWriter.c b/core/Lucy/Index/DocWriter.c
index ec583cc..43ac738 100644
--- a/core/Lucy/Index/DocWriter.c
+++ b/core/Lucy/Index/DocWriter.c
@@ -105,7 +105,7 @@ DocWriter_Add_Inverted_Doc_IMP(DocWriter *self, Inverter *inverter,
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Stored(type)) { num_stored++; }
}
- OutStream_Write_C32(dat_out, num_stored);
+ OutStream_Write_CU32(dat_out, num_stored);
Inverter_Iterate(inverter);
while (Inverter_Next(inverter)) {
@@ -140,12 +140,12 @@ DocWriter_Add_Inverted_Doc_IMP(DocWriter *self, Inverter *inverter,
}
case FType_INT32: {
int32_t val = (int32_t)Int_Get_Value((Integer*)value);
- OutStream_Write_C32(dat_out, val);
+ OutStream_Write_CI32(dat_out, val);
break;
}
case FType_INT64: {
int64_t val = Int_Get_Value((Integer*)value);
- OutStream_Write_C64(dat_out, val);
+ OutStream_Write_CI64(dat_out, val);
break;
}
case FType_FLOAT32: {
http://git-wip-us.apache.org/repos/asf/lucy/blob/febd2ff3/core/Lucy/Index/HighlightWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/HighlightWriter.c b/core/Lucy/Index/HighlightWriter.c
index d81b623..72153b2 100644
--- a/core/Lucy/Index/HighlightWriter.c
+++ b/core/Lucy/Index/HighlightWriter.c
@@ -121,7 +121,7 @@ HLWriter_Add_Inverted_Doc_IMP(HighlightWriter *self, Inverter *inverter,
num_highlightable++;
}
}
- OutStream_Write_C32(dat_out, num_highlightable);
+ OutStream_Write_CU32(dat_out, num_highlightable);
Inverter_Iterate(inverter);
while (Inverter_Next(inverter)) {
http://git-wip-us.apache.org/repos/asf/lucy/blob/febd2ff3/core/Lucy/Index/Posting/MatchPosting.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Posting/MatchPosting.c b/core/Lucy/Index/Posting/MatchPosting.c
index e44eee8..cfe9efd 100644
--- a/core/Lucy/Index/Posting/MatchPosting.c
+++ b/core/Lucy/Index/Posting/MatchPosting.c
@@ -272,7 +272,7 @@ MatchTInfoStepper_Write_Key_Frame_IMP(MatchTermInfoStepper *self,
TermInfoIVARS *const tinfo_ivars = TInfo_IVARS((TermInfo*)value);
// Write doc_freq.
- OutStream_Write_C32(outstream, doc_freq);
+ OutStream_Write_CI32(outstream, doc_freq);
// Write postings file pointer.
OutStream_Write_CI64(outstream, tinfo_ivars->post_filepos);
@@ -316,7 +316,7 @@ MatchTInfoStepper_Read_Key_Frame_IMP(MatchTermInfoStepper *self,
TermInfoIVARS *const tinfo_ivars = TInfo_IVARS((TermInfo*)ivars->value);
// Read doc freq.
- tinfo_ivars->doc_freq = InStream_Read_C32(instream);
+ tinfo_ivars->doc_freq = InStream_Read_CI32(instream);
// Read postings file pointer.
tinfo_ivars->post_filepos = InStream_Read_CI64(instream);
@@ -336,7 +336,7 @@ MatchTInfoStepper_Read_Delta_IMP(MatchTermInfoStepper *self, InStream *instream)
TermInfoIVARS *const tinfo_ivars = TInfo_IVARS((TermInfo*)ivars->value);
// Read doc freq.
- tinfo_ivars->doc_freq = InStream_Read_C32(instream);
+ tinfo_ivars->doc_freq = InStream_Read_CI32(instream);
// Adjust postings file pointer.
tinfo_ivars->post_filepos += InStream_Read_CI64(instream);
http://git-wip-us.apache.org/repos/asf/lucy/blob/febd2ff3/core/Lucy/Plan/TextType.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Plan/TextType.c b/core/Lucy/Plan/TextType.c
index 67f64a3..318fa47 100644
--- a/core/Lucy/Plan/TextType.c
+++ b/core/Lucy/Plan/TextType.c
@@ -135,7 +135,7 @@ TextTermStepper_Write_Delta_IMP(TextTermStepper *self, OutStream *outstream,
const size_t diff_len = new_size - overlap;
// Write number of common bytes and common bytes.
- OutStream_Write_C32(outstream, overlap);
+ OutStream_Write_CI32(outstream, overlap);
OutStream_Write_String(outstream, diff_start_str, diff_len);
// Update value.
http://git-wip-us.apache.org/repos/asf/lucy/blob/febd2ff3/core/Lucy/Search/SortRule.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Search/SortRule.c b/core/Lucy/Search/SortRule.c
index 736205c..dfb650c 100644
--- a/core/Lucy/Search/SortRule.c
+++ b/core/Lucy/Search/SortRule.c
@@ -63,22 +63,22 @@ SortRule_Destroy_IMP(SortRule *self) {
SortRule*
SortRule_Deserialize_IMP(SortRule *self, InStream *instream) {
SortRuleIVARS *ivars = SortRule_IVARS(self);
- ivars->type = InStream_Read_C32(instream);
+ ivars->type = InStream_Read_CI32(instream);
if (ivars->type == SortRule_FIELD) {
ivars->field = Freezer_read_string(instream);
}
- ivars->reverse = !!InStream_Read_C32(instream);
+ ivars->reverse = !!InStream_Read_CU32(instream);
return self;
}
void
SortRule_Serialize_IMP(SortRule *self, OutStream *target) {
SortRuleIVARS *ivars = SortRule_IVARS(self);
- OutStream_Write_C32(target, ivars->type);
+ OutStream_Write_CI32(target, ivars->type);
if (ivars->type == SortRule_FIELD) {
Freezer_serialize_string(ivars->field, target);
}
- OutStream_Write_C32(target, !!ivars->reverse);
+ OutStream_Write_CU32(target, !!ivars->reverse);
}
String*
http://git-wip-us.apache.org/repos/asf/lucy/blob/febd2ff3/core/Lucy/Search/TopDocs.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Search/TopDocs.c b/core/Lucy/Search/TopDocs.c
index ddaeb13..b685048 100644
--- a/core/Lucy/Search/TopDocs.c
+++ b/core/Lucy/Search/TopDocs.c
@@ -51,14 +51,14 @@ void
TopDocs_Serialize_IMP(TopDocs *self, OutStream *outstream) {
TopDocsIVARS *const ivars = TopDocs_IVARS(self);
Freezer_serialize_varray(ivars->match_docs, outstream);
- OutStream_Write_C32(outstream, ivars->total_hits);
+ OutStream_Write_CU32(outstream, ivars->total_hits);
}
TopDocs*
TopDocs_Deserialize_IMP(TopDocs *self, InStream *instream) {
TopDocsIVARS *const ivars = TopDocs_IVARS(self);
ivars->match_docs = Freezer_read_varray(instream);
- ivars->total_hits = InStream_Read_C32(instream);
+ ivars->total_hits = InStream_Read_CU32(instream);
return self;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/febd2ff3/core/Lucy/Util/Freezer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/Freezer.c b/core/Lucy/Util/Freezer.c
index 90a63c3..26039d2 100644
--- a/core/Lucy/Util/Freezer.c
+++ b/core/Lucy/Util/Freezer.c
@@ -68,7 +68,7 @@ Freezer_serialize(Obj *obj, OutStream *outstream) {
}
else if (Obj_is_a(obj, INTEGER)) {
int64_t val = Int_Get_Value((Integer*)obj);
- OutStream_Write_C64(outstream, (uint64_t)val);
+ OutStream_Write_CI64(outstream, val);
}
else if (Obj_is_a(obj, FLOAT)) {
double val = Float_Get_Value((Float*)obj);
@@ -241,13 +241,13 @@ Freezer_serialize_varray(Vector *array, OutStream *outstream) {
for (uint32_t i = 0; i < size; i++) {
Obj *elem = Vec_Fetch(array, i);
if (elem) {
- OutStream_Write_C32(outstream, i - last_valid_tick);
+ OutStream_Write_CU32(outstream, i - last_valid_tick);
FREEZE(elem, outstream);
last_valid_tick = i;
}
}
// Terminate.
- OutStream_Write_C32(outstream, size - last_valid_tick);
+ OutStream_Write_CU32(outstream, size - last_valid_tick);
}
Vector*
http://git-wip-us.apache.org/repos/asf/lucy/blob/febd2ff3/core/LucyX/Search/ProximityQuery.c
----------------------------------------------------------------------
diff --git a/core/LucyX/Search/ProximityQuery.c b/core/LucyX/Search/ProximityQuery.c
index 637fcce..1a93584 100644
--- a/core/LucyX/Search/ProximityQuery.c
+++ b/core/LucyX/Search/ProximityQuery.c
@@ -87,7 +87,7 @@ ProximityQuery_Serialize_IMP(ProximityQuery *self, OutStream *outstream) {
OutStream_Write_F32(outstream, ivars->boost);
Freezer_serialize_string(ivars->field, outstream);
Freezer_serialize_varray(ivars->terms, outstream);
- OutStream_Write_C32(outstream, ivars->within);
+ OutStream_Write_CU32(outstream, ivars->within);
}
ProximityQuery*
@@ -264,7 +264,7 @@ ProximityCompiler_Serialize_IMP(ProximityCompiler *self,
OutStream_Write_F32(outstream, ivars->raw_weight);
OutStream_Write_F32(outstream, ivars->query_norm_factor);
OutStream_Write_F32(outstream, ivars->normalized_weight);
- OutStream_Write_C32(outstream, ivars->within);
+ OutStream_Write_CU32(outstream, ivars->within);
}
ProximityCompiler*
@@ -278,7 +278,7 @@ ProximityCompiler_Deserialize_IMP(ProximityCompiler *self,
ivars->raw_weight = InStream_Read_F32(instream);
ivars->query_norm_factor = InStream_Read_F32(instream);
ivars->normalized_weight = InStream_Read_F32(instream);
- ivars->within = InStream_Read_C32(instream);
+ ivars->within = InStream_Read_CU32(instream);
return self;
}
[04/18] lucy git commit: Support compressed signed integers in
NumberUtils.
Posted by ma...@apache.org.
Support compressed signed integers in NumberUtils.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/cef8cbc1
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/cef8cbc1
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/cef8cbc1
Branch: refs/heads/master
Commit: cef8cbc1dc507196ac0881f72709031b2889abff
Parents: ce89b9c
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Mon Apr 18 17:12:14 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Mon Apr 18 17:12:14 2016 -0700
----------------------------------------------------------------------
core/Lucy/Test/Util/TestNumberUtils.c | 104 ++++++++++++++++++++++++++++-
core/Lucy/Util/NumberUtils.cfh | 52 +++++++++++++++
2 files changed, 155 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/cef8cbc1/core/Lucy/Test/Util/TestNumberUtils.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Test/Util/TestNumberUtils.c b/core/Lucy/Test/Util/TestNumberUtils.c
index ffd3079..125f325 100644
--- a/core/Lucy/Test/Util/TestNumberUtils.c
+++ b/core/Lucy/Test/Util/TestNumberUtils.c
@@ -94,6 +94,55 @@ test_u4(TestBatchRunner *runner) {
}
static void
+test_ci32(TestBatchRunner *runner) {
+ int64_t mins[] = { -500, -0x4000 - 100, INT32_MIN };
+ int64_t limits[] = { 500, -0x4000 + 100, INT32_MIN + 10 };
+ int32_t set_num;
+ int32_t num_sets = sizeof(mins) / sizeof(int64_t);
+ size_t count = 64;
+ int64_t *ints = NULL;
+ size_t amount = count * CI32_MAX_BYTES;
+ char *encoded = (char*)CALLOCATE(amount, sizeof(char));
+ char *target = encoded;
+ char *limit = target + amount;
+ const char *decode;
+
+ for (set_num = 0; set_num < num_sets; set_num++) {
+ const char *skip;
+ ints = TestUtils_random_i64s(ints, count,
+ mins[set_num], limits[set_num]);
+ target = encoded;
+ for (size_t i = 0; i < count; i++) {
+ NumUtil_encode_ci32((int32_t)ints[i], &target);
+ }
+ decode = encoded;
+ skip = encoded;
+ for (size_t i = 0; i < count; i++) {
+ TEST_INT_EQ(runner, NumUtil_decode_ci32(&decode), ints[i],
+ "ci32 %" PRId64, ints[i]);
+ NumUtil_skip_cint(&skip);
+ if (decode > limit) { THROW(ERR, "overrun"); }
+ }
+ TEST_TRUE(runner, skip == decode, "skip %" PRIu64 " == %" PRIu64,
+ (uint64_t)skip, (uint64_t)decode);
+ }
+
+ target = encoded;
+ NumUtil_encode_ci32(INT32_MAX, &target);
+ decode = encoded;
+ TEST_INT_EQ(runner, NumUtil_decode_ci32(&decode), INT32_MAX,
+ "ci32 INT32_MAX");
+ target = encoded;
+ NumUtil_encode_ci32(INT32_MIN, &target);
+ decode = encoded;
+ TEST_INT_EQ(runner, NumUtil_decode_ci32(&decode), INT32_MIN,
+ "ci32 INT32_MIN");
+
+ FREEMEM(encoded);
+ FREEMEM(ints);
+}
+
+static void
test_cu32(TestBatchRunner *runner) {
uint64_t mins[] = { 0, 0x4000 - 100, (uint32_t)INT32_MAX - 100, UINT32_MAX - 10 };
uint64_t limits[] = { 500, 0x4000 + 100, (uint32_t)INT32_MAX + 100, UINT32_MAX };
@@ -156,6 +205,57 @@ test_cu32(TestBatchRunner *runner) {
}
static void
+test_ci64(TestBatchRunner *runner) {
+ int64_t mins[] = { -500, -0x4000 - 100, (int64_t)INT32_MIN - 100, INT64_MIN };
+ int64_t limits[] = { 500, -0x4000 + 100, (int64_t)INT32_MIN + 1000, INT64_MIN + 10 };
+ int32_t set_num;
+ int32_t num_sets = sizeof(mins) / sizeof(int64_t);
+ size_t count = 64;
+ int64_t *ints = NULL;
+ size_t amount = count * CI64_MAX_BYTES;
+ char *encoded = (char*)CALLOCATE(amount, sizeof(char));
+ char *target = encoded;
+ char *limit = target + amount;
+ const char *decode;
+
+ for (set_num = 0; set_num < num_sets; set_num++) {
+ const char *skip;
+ ints = TestUtils_random_i64s(ints, count,
+ mins[set_num], limits[set_num]);
+ target = encoded;
+ for (size_t i = 0; i < count; i++) {
+ NumUtil_encode_ci64(ints[i], &target);
+ }
+ decode = encoded;
+ skip = encoded;
+ for (size_t i = 0; i < count; i++) {
+ int64_t got = NumUtil_decode_ci64(&decode);
+ TEST_INT_EQ(runner, got, ints[i],
+ "ci64 %" PRId64 " == %" PRId64, got, ints[i]);
+ if (decode > limit) { THROW(ERR, "overrun"); }
+ NumUtil_skip_cint(&skip);
+ }
+ TEST_TRUE(runner, skip == decode, "skip %lu == %lu",
+ (unsigned long)skip, (unsigned long)decode);
+ }
+
+ target = encoded;
+ NumUtil_encode_ci64(INT64_MAX, &target);
+ decode = encoded;
+ int64_t got = NumUtil_decode_ci64(&decode);
+ TEST_INT_EQ(runner, got, INT64_MAX, "ci64 INT64_MAX");
+
+ target = encoded;
+ NumUtil_encode_ci64(INT64_MIN, &target);
+ decode = encoded;
+ got = NumUtil_decode_ci64(&decode);
+ TEST_INT_EQ(runner, got, INT64_MIN, "ci64 INT64_MIN");
+
+ FREEMEM(encoded);
+ FREEMEM(ints);
+}
+
+static void
test_cu64(TestBatchRunner *runner) {
uint64_t mins[] = { 0, 0x4000 - 100, (uint64_t)UINT32_MAX - 100, UINT64_MAX - 10 };
uint64_t limits[] = { 500, 0x4000 + 100, (uint64_t)UINT32_MAX + 1000, UINT64_MAX };
@@ -359,12 +459,14 @@ test_bigend_f64(TestBatchRunner *runner) {
void
TestNumUtil_Run_IMP(TestNumberUtils *self, TestBatchRunner *runner) {
- TestBatchRunner_Plan(runner, (TestBatch*)self, 1196);
+ TestBatchRunner_Plan(runner, (TestBatch*)self, 1655);
srand((unsigned int)time((time_t*)NULL));
test_u1(runner);
test_u2(runner);
test_u4(runner);
+ test_ci32(runner);
test_cu32(runner);
+ test_ci64(runner);
test_cu64(runner);
test_bigend_u16(runner);
test_bigend_u32(runner);
http://git-wip-us.apache.org/repos/asf/lucy/blob/cef8cbc1/core/Lucy/Util/NumberUtils.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/NumberUtils.cfh b/core/Lucy/Util/NumberUtils.cfh
index 3d9c79d..8d1b0b1 100644
--- a/core/Lucy/Util/NumberUtils.cfh
+++ b/core/Lucy/Util/NumberUtils.cfh
@@ -87,6 +87,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline void
encode_c32(uint32_t value, char **dest);
+ /** Encode a compressed 32-bit signed integer at the space pointed to by
+ * `dest`. As a side effect, `dest` will be advanced to immediately after
+ * the end of the compressed data.
+ */
+ inert inline void
+ encode_ci32(int32_t value, char **dest);
+
/** Encode a compressed 32-bit unsigned integer at the space pointed to by
* `dest`. As a side effect, `dest` will be advanced to immediately after
* the end of the compressed data.
@@ -117,6 +124,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline void
encode_c64(uint64_t value, char **dest);
+ /** Encode a compressed 64-bit signed integer at the space pointed to by
+ * `dest`. As a side effect, `dest` will be advanced to immediately after
+ * the end of the compressed data.
+ */
+ inert inline void
+ encode_ci64(int64_t value, char **dest);
+
/** Encode a compressed 64-bit unsigned integer at the space pointed to by
* `dest`. As a side effect, `dest` will be advanced to immediately after
* the end of the compressed data.
@@ -131,6 +145,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline uint32_t
decode_c32(const char **source);
+ /** Read a compressed 32-bit signed integer from the buffer pointed to
+ * by `source`. As a side effect, advance the pointer, consuming the
+ * bytes occupied by the compressed number.
+ */
+ inert inline int32_t
+ decode_ci32(const char **source);
+
/** Read a compressed 32-bit unsigned integer from the buffer pointed to
* by `source`. As a side effect, advance the pointer, consuming the
* bytes occupied by the compressed number.
@@ -145,6 +166,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline uint64_t
decode_c64(const char **source);
+ /** Read a compressed 64-bit signed integer from the buffer pointed to
+ * by `source`. As a side effect, advance the pointer, consuming the
+ * bytes occupied by the compressed number.
+ */
+ inert inline int64_t
+ decode_ci64(const char **source);
+
/** Read a compressed 64-bit unsigned integer from the buffer pointed to
* by `source`. As a side effect, advance the pointer, consuming the
* bytes occupied by the compressed number.
@@ -332,7 +360,9 @@ lucy_NumUtil_decode_bigend_f64(const void *source) {
#define LUCY_NUMUTIL_C32_MAX_BYTES ((sizeof(uint32_t) * 8 / 7) + 1) /* 5 */
#define LUCY_NUMUTIL_C64_MAX_BYTES ((sizeof(uint64_t) * 8 / 7) + 1) /* 10 */
+#define LUCY_NUMUTIL_CI32_MAX_BYTES ((sizeof(int32_t) * 8 / 7) + 1) /* 5 */
#define LUCY_NUMUTIL_CU32_MAX_BYTES ((sizeof(uint32_t) * 8 / 7) + 1) /* 5 */
+#define LUCY_NUMUTIL_CI64_MAX_BYTES ((sizeof(int64_t) * 8 / 7) + 1) /* 10 */
#define LUCY_NUMUTIL_CU64_MAX_BYTES ((sizeof(uint64_t) * 8 / 7) + 1) /* 10 */
static CFISH_INLINE void
@@ -341,6 +371,11 @@ lucy_NumUtil_encode_c32(uint32_t value, char **out_buf) {
}
static CFISH_INLINE void
+lucy_NumUtil_encode_ci32(int32_t value, char **out_buf) {
+ lucy_NumUtil_encode_cu32((uint32_t)value, out_buf);
+}
+
+static CFISH_INLINE void
lucy_NumUtil_encode_cu32(uint32_t value, char **out_buf) {
uint8_t buf[LUCY_NUMUTIL_CU32_MAX_BYTES];
uint8_t *const limit = buf + sizeof(buf);
@@ -365,6 +400,11 @@ lucy_NumUtil_encode_c64(uint64_t value, char **out_buf) {
}
static CFISH_INLINE void
+lucy_NumUtil_encode_ci64(int64_t value, char **out_buf) {
+ lucy_NumUtil_encode_cu64((uint64_t)value, out_buf);
+}
+
+static CFISH_INLINE void
lucy_NumUtil_encode_cu64(uint64_t value, char **out_buf) {
uint8_t buf[LUCY_NUMUTIL_CU64_MAX_BYTES];
uint8_t *const limit = buf + sizeof(buf);
@@ -424,6 +464,11 @@ lucy_NumUtil_decode_c32(const char **source_ptr) {
return decoded;
}
+static CFISH_INLINE int32_t
+lucy_NumUtil_decode_ci32(const char **source_ptr) {
+ return (int32_t)lucy_NumUtil_decode_cu32(source_ptr);
+}
+
static CFISH_INLINE uint32_t
lucy_NumUtil_decode_cu32(const char **source_ptr) {
const char *source = *source_ptr;
@@ -442,6 +487,11 @@ lucy_NumUtil_decode_c64(const char **source_ptr) {
return decoded;
}
+static CFISH_INLINE int64_t
+lucy_NumUtil_decode_ci64(const char **source_ptr) {
+ return (int64_t)lucy_NumUtil_decode_cu64(source_ptr);
+}
+
static CFISH_INLINE uint64_t
lucy_NumUtil_decode_cu64(const char **source_ptr) {
const char *source = *source_ptr;
@@ -530,6 +580,8 @@ lucy_NumUtil_u4set(void *array, uint32_t tick, uint8_t value) {
#ifdef LUCY_USE_SHORT_NAMES
#define C32_MAX_BYTES LUCY_NUMUTIL_C32_MAX_BYTES
#define C64_MAX_BYTES LUCY_NUMUTIL_C64_MAX_BYTES
+ #define CI32_MAX_BYTES LUCY_NUMUTIL_CI32_MAX_BYTES
+ #define CI64_MAX_BYTES LUCY_NUMUTIL_CI64_MAX_BYTES
#define CU32_MAX_BYTES LUCY_NUMUTIL_CU32_MAX_BYTES
#define CU64_MAX_BYTES LUCY_NUMUTIL_CU64_MAX_BYTES
#endif
[11/18] lucy git commit: Change C32/C64 for freq, positions.
Posted by ma...@apache.org.
Change C32/C64 for freq, positions.
`freq` (num occurrences per document for a term) is generally
represented as a uint32_t. Positions may be signed or unsigned -- it's
not consistent. Just match up to existing usage.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/656a773b
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/656a773b
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/656a773b
Branch: refs/heads/master
Commit: 656a773b89d3676dc9d62d48f5a2d145342bb32c
Parents: 3089bfd
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Wed Apr 20 18:52:32 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Wed Apr 20 19:26:48 2016 -0700
----------------------------------------------------------------------
core/Lucy/Index/Posting/MatchPosting.c | 6 +++---
core/Lucy/Index/Posting/RawPosting.c | 2 +-
core/Lucy/Index/Posting/RichPosting.c | 4 ++--
core/Lucy/Index/TermVector.c | 12 ++++++------
4 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/656a773b/core/Lucy/Index/Posting/MatchPosting.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Posting/MatchPosting.c b/core/Lucy/Index/Posting/MatchPosting.c
index 2bda909..e44eee8 100644
--- a/core/Lucy/Index/Posting/MatchPosting.c
+++ b/core/Lucy/Index/Posting/MatchPosting.c
@@ -90,7 +90,7 @@ MatchPost_Read_Record_IMP(MatchPosting *self, InStream *instream) {
ivars->freq = 1;
}
else {
- ivars->freq = InStream_Read_C32(instream);
+ ivars->freq = InStream_Read_CU32(instream);
}
}
@@ -217,7 +217,7 @@ MatchPostWriter_Write_Posting_IMP(MatchPostingWriter *self, RawPosting *posting)
else {
const uint32_t doc_code = delta_doc << 1;
OutStream_Write_CU32(outstream, doc_code);
- OutStream_Write_C32(outstream, posting_ivars->freq);
+ OutStream_Write_CU32(outstream, posting_ivars->freq);
}
OutStream_Write_Bytes(outstream, aux_content, posting_ivars->aux_len);
ivars->last_doc_id = doc_id;
@@ -296,7 +296,7 @@ MatchTInfoStepper_Write_Delta_IMP(MatchTermInfoStepper *self,
- TInfo_IVARS(last_tinfo)->post_filepos;
// Write doc_freq.
- OutStream_Write_C32(outstream, doc_freq);
+ OutStream_Write_CU32(outstream, doc_freq);
// Write postings file pointer delta.
OutStream_Write_CI64(outstream, post_delta);
http://git-wip-us.apache.org/repos/asf/lucy/blob/656a773b/core/Lucy/Index/Posting/RawPosting.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Posting/RawPosting.c b/core/Lucy/Index/Posting/RawPosting.c
index 052020a..0fc5b2e 100644
--- a/core/Lucy/Index/Posting/RawPosting.c
+++ b/core/Lucy/Index/Posting/RawPosting.c
@@ -138,7 +138,7 @@ RawPostWriter_Write_Posting_IMP(RawPostingWriter *self, RawPosting *posting) {
else {
const uint32_t doc_code = delta_doc << 1;
OutStream_Write_CU32(outstream, doc_code);
- OutStream_Write_C32(outstream, posting_ivars->freq);
+ OutStream_Write_CU32(outstream, posting_ivars->freq);
}
OutStream_Write_Bytes(outstream, aux_content, posting_ivars->aux_len);
ivars->last_doc_id = doc_id;
http://git-wip-us.apache.org/repos/asf/lucy/blob/656a773b/core/Lucy/Index/Posting/RichPosting.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Posting/RichPosting.c b/core/Lucy/Index/Posting/RichPosting.c
index db590d4..da9da77 100644
--- a/core/Lucy/Index/Posting/RichPosting.c
+++ b/core/Lucy/Index/Posting/RichPosting.c
@@ -81,7 +81,7 @@ RichPost_Read_Record_IMP(RichPosting *self, InStream *instream) {
}
// Otherwise, freq was stored as a C32.
else {
- ivars->freq = InStream_Read_C32(instream);
+ ivars->freq = InStream_Read_CU32(instream);
}
// Read positions, aggregate per-position boost byte into weight.
@@ -96,7 +96,7 @@ RichPost_Read_Record_IMP(RichPosting *self, InStream *instream) {
float *prox_boosts = ivars->prox_boosts;
while (num_prox--) {
- position += InStream_Read_C32(instream);
+ position += InStream_Read_CU32(instream);
*positions++ = position;
*prox_boosts = norm_decoder[InStream_Read_U8(instream)];
aggregate_weight += *prox_boosts;
http://git-wip-us.apache.org/repos/asf/lucy/blob/656a773b/core/Lucy/Index/TermVector.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/TermVector.c b/core/Lucy/Index/TermVector.c
index 9c3a6f9..8ad0bcf 100644
--- a/core/Lucy/Index/TermVector.c
+++ b/core/Lucy/Index/TermVector.c
@@ -93,9 +93,9 @@ TV_Serialize_IMP(TermVector *self, OutStream *target) {
OutStream_Write_CU64(target, ivars->num_pos);
for (size_t i = 0; i < ivars->num_pos; i++) {
- OutStream_Write_C32(target, posits[i]);
- OutStream_Write_C32(target, starts[i]);
- OutStream_Write_C32(target, ends[i]);
+ OutStream_Write_CI32(target, posits[i]);
+ OutStream_Write_CI32(target, starts[i]);
+ OutStream_Write_CI32(target, ends[i]);
}
}
@@ -110,9 +110,9 @@ TV_Deserialize_IMP(TermVector *self, InStream *instream) {
int32_t *starts = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
int32_t *ends = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
for (size_t i = 0; i < num_pos; i++) {
- posits[i] = InStream_Read_C32(instream);
- starts[i] = InStream_Read_C32(instream);
- ends[i] = InStream_Read_C32(instream);
+ posits[i] = InStream_Read_CI32(instream);
+ starts[i] = InStream_Read_CI32(instream);
+ ends[i] = InStream_Read_CI32(instream);
}
I32Array *positions = I32Arr_new_steal(posits, num_pos);
I32Array *start_offsets = I32Arr_new_steal(starts, num_pos);
[10/18] lucy git commit: Switch C32 code relating to doc IDs.
Posted by ma...@apache.org.
Switch C32 code relating to doc IDs.
Doc IDs are signed. Delta doc IDs may signed or unsigned. Use the
appropriate CI32 or CU32 read/write methods for both.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/3089bfdf
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/3089bfdf
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/3089bfdf
Branch: refs/heads/master
Commit: 3089bfdf4c09b7cf3f937405417c93e23063d727
Parents: e5ba8a1
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Wed Apr 20 18:40:26 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Wed Apr 20 19:26:48 2016 -0700
----------------------------------------------------------------------
c/src/Lucy/Document/Doc.c | 4 ++--
core/Lucy/Index/Posting/MatchPosting.c | 4 ++--
core/Lucy/Index/Posting/RawPosting.c | 4 ++--
core/Lucy/Index/SkipStepper.c | 4 ++--
core/Lucy/Search/MatchDoc.c | 4 ++--
5 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/3089bfdf/c/src/Lucy/Document/Doc.c
----------------------------------------------------------------------
diff --git a/c/src/Lucy/Document/Doc.c b/c/src/Lucy/Document/Doc.c
index d0fb065..19b9d9d 100644
--- a/c/src/Lucy/Document/Doc.c
+++ b/c/src/Lucy/Document/Doc.c
@@ -69,14 +69,14 @@ Doc_Serialize_IMP(Doc *self, OutStream *outstream) {
DocIVARS *const ivars = Doc_IVARS(self);
Hash *hash = (Hash*)ivars->fields;
Freezer_serialize_hash(hash, outstream);
- OutStream_Write_C32(outstream, ivars->doc_id);
+ OutStream_Write_CI32(outstream, ivars->doc_id);
}
Doc*
Doc_Deserialize_IMP(Doc *self, InStream *instream) {
DocIVARS *const ivars = Doc_IVARS(self);
ivars->fields = Freezer_read_hash(instream);
- ivars->doc_id = InStream_Read_C32(instream);
+ ivars->doc_id = InStream_Read_CI32(instream);
return self;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/3089bfdf/core/Lucy/Index/Posting/MatchPosting.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Posting/MatchPosting.c b/core/Lucy/Index/Posting/MatchPosting.c
index c7348fa..2bda909 100644
--- a/core/Lucy/Index/Posting/MatchPosting.c
+++ b/core/Lucy/Index/Posting/MatchPosting.c
@@ -212,11 +212,11 @@ MatchPostWriter_Write_Posting_IMP(MatchPostingWriter *self, RawPosting *posting)
+ posting_ivars->content_len;
if (posting_ivars->freq == 1) {
const uint32_t doc_code = (delta_doc << 1) | 1;
- OutStream_Write_C32(outstream, doc_code);
+ OutStream_Write_CU32(outstream, doc_code);
}
else {
const uint32_t doc_code = delta_doc << 1;
- OutStream_Write_C32(outstream, doc_code);
+ OutStream_Write_CU32(outstream, doc_code);
OutStream_Write_C32(outstream, posting_ivars->freq);
}
OutStream_Write_Bytes(outstream, aux_content, posting_ivars->aux_len);
http://git-wip-us.apache.org/repos/asf/lucy/blob/3089bfdf/core/Lucy/Index/Posting/RawPosting.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Posting/RawPosting.c b/core/Lucy/Index/Posting/RawPosting.c
index 06e5367..052020a 100644
--- a/core/Lucy/Index/Posting/RawPosting.c
+++ b/core/Lucy/Index/Posting/RawPosting.c
@@ -133,11 +133,11 @@ RawPostWriter_Write_Posting_IMP(RawPostingWriter *self, RawPosting *posting) {
+ posting_ivars->content_len;
if (posting_ivars->freq == 1) {
const uint32_t doc_code = (delta_doc << 1) | 1;
- OutStream_Write_C32(outstream, doc_code);
+ OutStream_Write_CU32(outstream, doc_code);
}
else {
const uint32_t doc_code = delta_doc << 1;
- OutStream_Write_C32(outstream, doc_code);
+ OutStream_Write_CU32(outstream, doc_code);
OutStream_Write_C32(outstream, posting_ivars->freq);
}
OutStream_Write_Bytes(outstream, aux_content, posting_ivars->aux_len);
http://git-wip-us.apache.org/repos/asf/lucy/blob/3089bfdf/core/Lucy/Index/SkipStepper.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SkipStepper.c b/core/Lucy/Index/SkipStepper.c
index 634e2d8..2fe5b3c 100644
--- a/core/Lucy/Index/SkipStepper.c
+++ b/core/Lucy/Index/SkipStepper.c
@@ -47,7 +47,7 @@ SkipStepper_Set_ID_And_Filepos_IMP(SkipStepper *self, int32_t doc_id,
void
SkipStepper_Read_Record_IMP(SkipStepper *self, InStream *instream) {
SkipStepperIVARS *const ivars = SkipStepper_IVARS(self);
- ivars->doc_id += InStream_Read_C32(instream);
+ ivars->doc_id += InStream_Read_CI32(instream);
ivars->filepos += InStream_Read_CI64(instream);
}
@@ -66,7 +66,7 @@ SkipStepper_Write_Record_IMP(SkipStepper *self, OutStream *outstream,
const int64_t delta_filepos = ivars->filepos - last_filepos;
// Write delta doc id.
- OutStream_Write_C32(outstream, delta_doc_id);
+ OutStream_Write_CI32(outstream, delta_doc_id);
// Write delta file pointer.
OutStream_Write_CI64(outstream, delta_filepos);
http://git-wip-us.apache.org/repos/asf/lucy/blob/3089bfdf/core/Lucy/Search/MatchDoc.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Search/MatchDoc.c b/core/Lucy/Search/MatchDoc.c
index bbc4b59..1ac57ed 100644
--- a/core/Lucy/Search/MatchDoc.c
+++ b/core/Lucy/Search/MatchDoc.c
@@ -47,7 +47,7 @@ MatchDoc_Destroy_IMP(MatchDoc *self) {
void
MatchDoc_Serialize_IMP(MatchDoc *self, OutStream *outstream) {
MatchDocIVARS *const ivars = MatchDoc_IVARS(self);
- OutStream_Write_C32(outstream, ivars->doc_id);
+ OutStream_Write_CI32(outstream, ivars->doc_id);
OutStream_Write_F32(outstream, ivars->score);
OutStream_Write_U8(outstream, ivars->values ? 1 : 0);
if (ivars->values) { Freezer_serialize_varray(ivars->values, outstream); }
@@ -56,7 +56,7 @@ MatchDoc_Serialize_IMP(MatchDoc *self, OutStream *outstream) {
MatchDoc*
MatchDoc_Deserialize_IMP(MatchDoc *self, InStream *instream) {
MatchDocIVARS *const ivars = MatchDoc_IVARS(self);
- ivars->doc_id = InStream_Read_C32(instream);
+ ivars->doc_id = InStream_Read_CI32(instream);
ivars->score = InStream_Read_F32(instream);
if (InStream_Read_U8(instream)) {
ivars->values = Freezer_read_varray(instream);
[16/18] lucy git commit: Invoke signed version of function.
Posted by ma...@apache.org.
Invoke signed version of function.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/7876b136
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/7876b136
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/7876b136
Branch: refs/heads/master
Commit: 7876b136436fe89419e86d7ad635abc1322864eb
Parents: ab5a44b
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Thu Apr 21 14:01:58 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Thu Apr 21 14:01:58 2016 -0700
----------------------------------------------------------------------
core/Lucy/Index/Posting/MatchPosting.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/7876b136/core/Lucy/Index/Posting/MatchPosting.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Posting/MatchPosting.c b/core/Lucy/Index/Posting/MatchPosting.c
index cfe9efd..c1530ea 100644
--- a/core/Lucy/Index/Posting/MatchPosting.c
+++ b/core/Lucy/Index/Posting/MatchPosting.c
@@ -296,7 +296,7 @@ MatchTInfoStepper_Write_Delta_IMP(MatchTermInfoStepper *self,
- TInfo_IVARS(last_tinfo)->post_filepos;
// Write doc_freq.
- OutStream_Write_CU32(outstream, doc_freq);
+ OutStream_Write_CI32(outstream, doc_freq);
// Write postings file pointer delta.
OutStream_Write_CI64(outstream, post_delta);
[18/18] lucy git commit: Merge branch 'LUCY-301-C32-transition'
Posted by ma...@apache.org.
Merge branch 'LUCY-301-C32-transition'
At present, InStream, OutStream, and NumberUtils provide an unsigned
compressed format called C32/C64 -- and when we need signed
representations, we just cast.
To get rid of all these casts (and fix some sites missing casts),
support signed and unsigned variants.
This closes #42.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/e6b61338
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/e6b61338
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/e6b61338
Branch: refs/heads/master
Commit: e6b61338ce163a89a4e34e2e230c5e6ea1bb437b
Parents: 8b4ad21 6c167aa
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Thu Apr 21 14:12:05 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Thu Apr 21 14:12:05 2016 -0700
----------------------------------------------------------------------
c/src/Lucy/Document/Doc.c | 4 +-
c/src/Lucy/Index/DocReader.c | 16 +--
core/Lucy/Index/DocVector.c | 18 +--
core/Lucy/Index/DocWriter.c | 18 ++-
core/Lucy/Index/HighlightReader.c | 2 +-
core/Lucy/Index/HighlightWriter.c | 28 ++---
core/Lucy/Index/LexIndex.c | 8 +-
core/Lucy/Index/LexiconWriter.c | 2 +-
core/Lucy/Index/Posting/MatchPosting.c | 38 +++----
core/Lucy/Index/Posting/RawPosting.c | 6 +-
core/Lucy/Index/Posting/RichPosting.c | 18 +--
core/Lucy/Index/Posting/ScorePosting.c | 22 ++--
core/Lucy/Index/SkipStepper.c | 8 +-
core/Lucy/Index/TermVector.c | 16 +--
core/Lucy/Plan/TextType.c | 11 +-
core/Lucy/Search/MatchDoc.c | 4 +-
core/Lucy/Search/SortRule.c | 8 +-
core/Lucy/Search/SortSpec.c | 5 +-
core/Lucy/Search/TopDocs.c | 4 +-
core/Lucy/Store/InStream.c | 30 ++++-
core/Lucy/Store/InStream.cfh | 25 +++--
core/Lucy/Store/OutStream.c | 38 +++++--
core/Lucy/Store/OutStream.cfh | 18 ++-
core/Lucy/Test/Store/TestIOPrimitives.c | 110 ++++++++++++++++---
core/Lucy/Test/Util/TestNumberUtils.c | 146 +++++++++++++++++++++----
core/Lucy/Util/Freezer.c | 40 ++++---
core/Lucy/Util/NumberUtils.cfh | 120 ++++++++++++++------
core/LucyX/Search/ProximityQuery.c | 8 +-
go/build.go | 12 +-
go/lucy/lucy.go | 16 +--
go/lucy/store.go | 50 +++++++--
go/lucy/store_test.go | 28 +++--
perl/buildlib/Lucy/Build/Binding/Store.pm | 7 +-
perl/lib/LucyX/Search/Filter.pm | 4 +-
perl/t/023-stepper.t | 4 +-
perl/t/102-strings_io.t | 2 +-
perl/t/binding/101-simple_io.t | 20 ++--
perl/xs/Lucy/Document/Doc.c | 8 +-
perl/xs/Lucy/Index/DocReader.c | 14 +--
39 files changed, 652 insertions(+), 284 deletions(-)
----------------------------------------------------------------------
[02/18] lucy git commit: Add explicit cu32 and cu64 to NumUtil.
Posted by ma...@apache.org.
Add explicit cu32 and cu64 to NumUtil.
Duplicate the functionality of xxxx_c64 in xxxx_cu64, in preparation for
the addition of xxxx_ci64 (for signed integers) and the eventual removal
of xxxx_64.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/3e3c6489
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/3e3c6489
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/3e3c6489
Branch: refs/heads/master
Commit: 3e3c64895bc92b19fadebe2a7b3188e8a218597d
Parents: 8a48ef5
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Mon Apr 18 15:07:21 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Mon Apr 18 15:12:10 2016 -0700
----------------------------------------------------------------------
core/Lucy/Test/Store/TestIOPrimitives.c | 2 +-
core/Lucy/Test/Util/TestNumberUtils.c | 42 +++++++--------
core/Lucy/Util/NumberUtils.cfh | 81 ++++++++++++++++++++++++++--
3 files changed, 99 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e3c6489/core/Lucy/Test/Store/TestIOPrimitives.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Test/Store/TestIOPrimitives.c b/core/Lucy/Test/Store/TestIOPrimitives.c
index 7409d7e..3c09dde 100644
--- a/core/Lucy/Test/Store/TestIOPrimitives.c
+++ b/core/Lucy/Test/Store/TestIOPrimitives.c
@@ -321,7 +321,7 @@ test_cu64(TestBatchRunner *runner) {
char buffer[10];
const char *buf = buffer;
size_t size = InStream_Read_Raw_C64(raw_instream, buffer);
- uint64_t got = NumUtil_decode_c64(&buf);
+ uint64_t got = NumUtil_decode_cu64(&buf);
UNUSED_VAR(size);
if (got != ints[i]) {
FAIL(runner, "Read_Raw_C64 failed: %" PRIu64 ", %" PRIu64,
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e3c6489/core/Lucy/Test/Util/TestNumberUtils.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Test/Util/TestNumberUtils.c b/core/Lucy/Test/Util/TestNumberUtils.c
index 70a63a6..ffd3079 100644
--- a/core/Lucy/Test/Util/TestNumberUtils.c
+++ b/core/Lucy/Test/Util/TestNumberUtils.c
@@ -94,14 +94,14 @@ test_u4(TestBatchRunner *runner) {
}
static void
-test_c32(TestBatchRunner *runner) {
+test_cu32(TestBatchRunner *runner) {
uint64_t mins[] = { 0, 0x4000 - 100, (uint32_t)INT32_MAX - 100, UINT32_MAX - 10 };
uint64_t limits[] = { 500, 0x4000 + 100, (uint32_t)INT32_MAX + 100, UINT32_MAX };
uint32_t set_num;
uint32_t num_sets = sizeof(mins) / sizeof(uint64_t);
size_t count = 64;
uint64_t *ints = NULL;
- size_t amount = count * C32_MAX_BYTES;
+ size_t amount = count * CU32_MAX_BYTES;
char *encoded = (char*)CALLOCATE(amount, sizeof(char));
char *target = encoded;
char *limit = target + amount;
@@ -114,13 +114,13 @@ test_c32(TestBatchRunner *runner) {
target = encoded;
for (size_t i = 0; i < count; i++) {
ints[i] = (uint32_t)ints[i];
- NumUtil_encode_c32((uint32_t)ints[i], &target);
+ NumUtil_encode_cu32((uint32_t)ints[i], &target);
}
decode = encoded;
skip = encoded;
for (size_t i = 0; i < count; i++) {
- TEST_INT_EQ(runner, NumUtil_decode_c32(&decode), ints[i],
- "c32 %lu", (long)ints[i]);
+ TEST_INT_EQ(runner, NumUtil_decode_cu32(&decode), ints[i],
+ "cu32 %lu", (long)ints[i]);
NumUtil_skip_cint(&skip);
if (decode > limit) { THROW(ERR, "overrun"); }
}
@@ -129,16 +129,16 @@ test_c32(TestBatchRunner *runner) {
target = encoded;
for (size_t i = 0; i < count; i++) {
- NumUtil_encode_padded_c32((uint32_t)ints[i], &target);
+ NumUtil_encode_padded_cu32((uint32_t)ints[i], &target);
}
TEST_TRUE(runner, target == limit,
- "padded c32 uses 5 bytes (%lu == %lu)", (unsigned long)target,
+ "padded cu32 uses 5 bytes (%lu == %lu)", (unsigned long)target,
(unsigned long)limit);
decode = encoded;
skip = encoded;
for (size_t i = 0; i < count; i++) {
- TEST_INT_EQ(runner, NumUtil_decode_c32(&decode), ints[i],
- "padded c32 %lu", (long)ints[i]);
+ TEST_INT_EQ(runner, NumUtil_decode_cu32(&decode), ints[i],
+ "padded cu32 %lu", (long)ints[i]);
NumUtil_skip_cint(&skip);
if (decode > limit) { THROW(ERR, "overrun"); }
}
@@ -147,23 +147,23 @@ test_c32(TestBatchRunner *runner) {
}
target = encoded;
- NumUtil_encode_c32(UINT32_MAX, &target);
+ NumUtil_encode_cu32(UINT32_MAX, &target);
decode = encoded;
- TEST_INT_EQ(runner, NumUtil_decode_c32(&decode), UINT32_MAX, "c32 UINT32_MAX");
+ TEST_INT_EQ(runner, NumUtil_decode_cu32(&decode), UINT32_MAX, "cu32 UINT32_MAX");
FREEMEM(encoded);
FREEMEM(ints);
}
static void
-test_c64(TestBatchRunner *runner) {
+test_cu64(TestBatchRunner *runner) {
uint64_t mins[] = { 0, 0x4000 - 100, (uint64_t)UINT32_MAX - 100, UINT64_MAX - 10 };
uint64_t limits[] = { 500, 0x4000 + 100, (uint64_t)UINT32_MAX + 1000, UINT64_MAX };
uint32_t set_num;
uint32_t num_sets = sizeof(mins) / sizeof(uint64_t);
size_t count = 64;
uint64_t *ints = NULL;
- size_t amount = count * C64_MAX_BYTES;
+ size_t amount = count * CU64_MAX_BYTES;
char *encoded = (char*)CALLOCATE(amount, sizeof(char));
char *target = encoded;
char *limit = target + amount;
@@ -175,14 +175,14 @@ test_c64(TestBatchRunner *runner) {
mins[set_num], limits[set_num]);
target = encoded;
for (size_t i = 0; i < count; i++) {
- NumUtil_encode_c64(ints[i], &target);
+ NumUtil_encode_cu64(ints[i], &target);
}
decode = encoded;
skip = encoded;
for (size_t i = 0; i < count; i++) {
- uint64_t got = NumUtil_decode_c64(&decode);
+ uint64_t got = NumUtil_decode_cu64(&decode);
TEST_TRUE(runner, got == ints[i],
- "c64 %" PRIu64 " == %" PRIu64, got, ints[i]);
+ "cu64 %" PRIu64 " == %" PRIu64, got, ints[i]);
if (decode > limit) { THROW(ERR, "overrun"); }
NumUtil_skip_cint(&skip);
}
@@ -191,11 +191,11 @@ test_c64(TestBatchRunner *runner) {
}
target = encoded;
- NumUtil_encode_c64(UINT64_MAX, &target);
+ NumUtil_encode_cu64(UINT64_MAX, &target);
decode = encoded;
- uint64_t got = NumUtil_decode_c64(&decode);
- TEST_TRUE(runner, got == UINT64_MAX, "c64 UINT64_MAX");
+ uint64_t got = NumUtil_decode_cu64(&decode);
+ TEST_TRUE(runner, got == UINT64_MAX, "cu64 UINT64_MAX");
FREEMEM(encoded);
FREEMEM(ints);
@@ -364,8 +364,8 @@ TestNumUtil_Run_IMP(TestNumberUtils *self, TestBatchRunner *runner) {
test_u1(runner);
test_u2(runner);
test_u4(runner);
- test_c32(runner);
- test_c64(runner);
+ test_cu32(runner);
+ test_cu64(runner);
test_bigend_u16(runner);
test_bigend_u32(runner);
test_bigend_u64(runner);
http://git-wip-us.apache.org/repos/asf/lucy/blob/3e3c6489/core/Lucy/Util/NumberUtils.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/NumberUtils.cfh b/core/Lucy/Util/NumberUtils.cfh
index b7c08b4..3d9c79d 100644
--- a/core/Lucy/Util/NumberUtils.cfh
+++ b/core/Lucy/Util/NumberUtils.cfh
@@ -87,6 +87,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline void
encode_c32(uint32_t value, char **dest);
+ /** Encode a compressed 32-bit unsigned integer at the space pointed to by
+ * `dest`. As a side effect, `dest` will be advanced to immediately after
+ * the end of the compressed data.
+ */
+ inert inline void
+ encode_cu32(uint32_t value, char **dest);
+
/** Encode a C32 at the space pointed to by `dest`, but add
* "leading zeroes" so that the space consumed will always be 5 bytes. As
* a side effect, `dest` will be advanced to immediately after
@@ -95,6 +102,14 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline void
encode_padded_c32(uint32_t value, char **dest);
+ /** Encode a compressed 32-bit unsigned integer at the space pointed to by
+ * `dest`, but add "leading zeroes" so that the space consumed will always
+ * be 5 bytes. As a side effect, `dest` will be advanced to immediately
+ * after the end of the compressed data.
+ */
+ inert inline void
+ encode_padded_cu32(uint32_t value, char **dest);
+
/** Encode a C64 at the space pointed to by `dest`. As a side
* effect, `dest` will be advanced to immediately after the end
* of the C64.
@@ -102,6 +117,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline void
encode_c64(uint64_t value, char **dest);
+ /** Encode a compressed 64-bit unsigned integer at the space pointed to by
+ * `dest`. As a side effect, `dest` will be advanced to immediately after
+ * the end of the compressed data.
+ */
+ inert inline void
+ encode_cu64(uint64_t value, char **dest);
+
/** Read a C32 from the buffer pointed to by `source`. As a
* side effect, advance the pointer, consuming the bytes occupied by the
* C32.
@@ -109,6 +131,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline uint32_t
decode_c32(const char **source);
+ /** Read a compressed 32-bit unsigned integer from the buffer pointed to
+ * by `source`. As a side effect, advance the pointer, consuming the
+ * bytes occupied by the compressed number.
+ */
+ inert inline uint32_t
+ decode_cu32(const char **source);
+
/** Read a C64 from the buffer pointed to by `source`. As a
* side effect, advance the pointer, consuming the bytes occupied by the
* C64.
@@ -116,6 +145,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline uint64_t
decode_c64(const char **source);
+ /** Read a compressed 64-bit unsigned integer from the buffer pointed to
+ * by `source`. As a side effect, advance the pointer, consuming the
+ * bytes occupied by the compressed number.
+ */
+ inert inline uint64_t
+ decode_cu64(const char **source);
+
/** Advance `source` past one encoded C32 or C64.
*/
inert inline void
@@ -296,10 +332,17 @@ lucy_NumUtil_decode_bigend_f64(const void *source) {
#define LUCY_NUMUTIL_C32_MAX_BYTES ((sizeof(uint32_t) * 8 / 7) + 1) /* 5 */
#define LUCY_NUMUTIL_C64_MAX_BYTES ((sizeof(uint64_t) * 8 / 7) + 1) /* 10 */
+#define LUCY_NUMUTIL_CU32_MAX_BYTES ((sizeof(uint32_t) * 8 / 7) + 1) /* 5 */
+#define LUCY_NUMUTIL_CU64_MAX_BYTES ((sizeof(uint64_t) * 8 / 7) + 1) /* 10 */
static CFISH_INLINE void
lucy_NumUtil_encode_c32(uint32_t value, char **out_buf) {
- uint8_t buf[LUCY_NUMUTIL_C32_MAX_BYTES];
+ lucy_NumUtil_encode_cu32(value, out_buf);
+}
+
+static CFISH_INLINE void
+lucy_NumUtil_encode_cu32(uint32_t value, char **out_buf) {
+ uint8_t buf[LUCY_NUMUTIL_CU32_MAX_BYTES];
uint8_t *const limit = buf + sizeof(buf);
uint8_t *ptr = limit - 1;
int num_bytes;
@@ -318,7 +361,12 @@ lucy_NumUtil_encode_c32(uint32_t value, char **out_buf) {
static CFISH_INLINE void
lucy_NumUtil_encode_c64(uint64_t value, char **out_buf) {
- uint8_t buf[LUCY_NUMUTIL_C64_MAX_BYTES];
+ lucy_NumUtil_encode_cu64(value, out_buf);
+}
+
+static CFISH_INLINE void
+lucy_NumUtil_encode_cu64(uint64_t value, char **out_buf) {
+ uint8_t buf[LUCY_NUMUTIL_CU64_MAX_BYTES];
uint8_t *const limit = buf + sizeof(buf);
uint8_t *ptr = limit - 1;
int num_bytes;
@@ -337,7 +385,12 @@ lucy_NumUtil_encode_c64(uint64_t value, char **out_buf) {
static CFISH_INLINE void
lucy_NumUtil_encode_padded_c32(uint32_t value, char **out_buf) {
- uint8_t buf[LUCY_NUMUTIL_C32_MAX_BYTES]
+ lucy_NumUtil_encode_padded_cu32(value, out_buf);
+}
+
+static CFISH_INLINE void
+lucy_NumUtil_encode_padded_cu32(uint32_t value, char **out_buf) {
+ uint8_t buf[LUCY_NUMUTIL_CU32_MAX_BYTES]
= { 0x80, 0x80, 0x80, 0x80, 0x80 };
uint8_t *const limit = buf + sizeof(buf);
uint8_t *ptr = limit - 1;
@@ -349,7 +402,7 @@ lucy_NumUtil_encode_padded_c32(uint32_t value, char **out_buf) {
*--ptr = ((value & 0x7f) | 0x80);
value >>= 7;
}
- memcpy(*out_buf, buf, LUCY_NUMUTIL_C32_MAX_BYTES);
+ memcpy(*out_buf, buf, LUCY_NUMUTIL_CU32_MAX_BYTES);
*out_buf += sizeof(buf);
}
@@ -371,6 +424,15 @@ lucy_NumUtil_decode_c32(const char **source_ptr) {
return decoded;
}
+static CFISH_INLINE uint32_t
+lucy_NumUtil_decode_cu32(const char **source_ptr) {
+ const char *source = *source_ptr;
+ uint32_t decoded;
+ LUCY_NUMUTIL_DECODE_CINT(decoded, source);
+ *source_ptr = source;
+ return decoded;
+}
+
static CFISH_INLINE uint64_t
lucy_NumUtil_decode_c64(const char **source_ptr) {
const char *source = *source_ptr;
@@ -380,6 +442,15 @@ lucy_NumUtil_decode_c64(const char **source_ptr) {
return decoded;
}
+static CFISH_INLINE uint64_t
+lucy_NumUtil_decode_cu64(const char **source_ptr) {
+ const char *source = *source_ptr;
+ uint64_t decoded;
+ LUCY_NUMUTIL_DECODE_CINT(decoded, source);
+ *source_ptr = source;
+ return decoded;
+}
+
static CFISH_INLINE void
lucy_NumUtil_skip_cint(const char **source_ptr) {
const uint8_t *ptr = *(const uint8_t**)source_ptr;
@@ -459,6 +530,8 @@ lucy_NumUtil_u4set(void *array, uint32_t tick, uint8_t value) {
#ifdef LUCY_USE_SHORT_NAMES
#define C32_MAX_BYTES LUCY_NUMUTIL_C32_MAX_BYTES
#define C64_MAX_BYTES LUCY_NUMUTIL_C64_MAX_BYTES
+ #define CU32_MAX_BYTES LUCY_NUMUTIL_CU32_MAX_BYTES
+ #define CU64_MAX_BYTES LUCY_NUMUTIL_CU64_MAX_BYTES
#endif
__END_C__
[05/18] lucy git commit: Switch NumUtils client code to cu32/ci32.
Posted by ma...@apache.org.
Switch NumUtils client code to cu32/ci32.
Change all invocations from NumUtil_xxxx_c32 to use the
NumUtil_xxxx_cu32 and NumUtil_xxx_ci32 variants which are explicitly
signed or unsigned.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/67656d94
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/67656d94
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/67656d94
Branch: refs/heads/master
Commit: 67656d94224b9350deb224699eda5956a0b8a989
Parents: cef8cbc
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Mon Apr 18 17:52:42 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Mon Apr 18 18:43:05 2016 -0700
----------------------------------------------------------------------
core/Lucy/Index/DocVector.c | 18 +++++++++---------
core/Lucy/Index/HighlightWriter.c | 26 +++++++++++++-------------
core/Lucy/Index/Posting/RichPosting.c | 8 ++++----
core/Lucy/Index/Posting/ScorePosting.c | 18 +++++++++---------
core/Lucy/Store/OutStream.c | 4 ++--
5 files changed, 37 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/67656d94/core/Lucy/Index/DocVector.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/DocVector.c b/core/Lucy/Index/DocVector.c
index 54dfedd..2bc8192 100644
--- a/core/Lucy/Index/DocVector.c
+++ b/core/Lucy/Index/DocVector.c
@@ -122,13 +122,13 @@ static Hash*
S_extract_tv_cache(Blob *field_buf) {
Hash *tv_cache = Hash_new(0);
const char *tv_string = Blob_Get_Buf(field_buf);
- int32_t num_terms = NumUtil_decode_c32(&tv_string);
+ int32_t num_terms = NumUtil_decode_ci32(&tv_string);
ByteBuf *text_buf = BB_new(0);
// Read the number of highlightable terms in the field.
for (int32_t i = 0; i < num_terms; i++) {
- size_t overlap = NumUtil_decode_c32(&tv_string);
- size_t len = NumUtil_decode_c32(&tv_string);
+ size_t overlap = NumUtil_decode_cu32(&tv_string);
+ size_t len = NumUtil_decode_cu32(&tv_string);
// Decompress the term text.
BB_Set_Size(text_buf, overlap);
@@ -137,7 +137,7 @@ S_extract_tv_cache(Blob *field_buf) {
// Get positions & offsets string.
const char *bookmark_ptr = tv_string;
- int32_t num_positions = NumUtil_decode_c32(&tv_string);
+ int32_t num_positions = NumUtil_decode_ci32(&tv_string);
while (num_positions--) {
// Leave nums compressed to save a little mem.
NumUtil_skip_cint(&tv_string);
@@ -167,17 +167,17 @@ S_extract_tv_from_tv_buf(String *field, String *term_text, Blob *tv_buf) {
uint32_t num_pos = 0;
if (posdata != posdata_end) {
- num_pos = NumUtil_decode_c32(&posdata);
+ num_pos = NumUtil_decode_cu32(&posdata);
positions = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
starts = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
ends = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
}
- // Expand C32s.
+ // Expand CI32s.
for (uint32_t i = 0; i < num_pos; i++) {
- positions[i] = NumUtil_decode_c32(&posdata);
- starts[i] = NumUtil_decode_c32(&posdata);
- ends[i] = NumUtil_decode_c32(&posdata);
+ positions[i] = NumUtil_decode_ci32(&posdata);
+ starts[i] = NumUtil_decode_ci32(&posdata);
+ ends[i] = NumUtil_decode_ci32(&posdata);
}
if (posdata != posdata_end) {
http://git-wip-us.apache.org/repos/asf/lucy/blob/67656d94/core/Lucy/Index/HighlightWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/HighlightWriter.c b/core/Lucy/Index/HighlightWriter.c
index 08ad85a..d81b623 100644
--- a/core/Lucy/Index/HighlightWriter.c
+++ b/core/Lucy/Index/HighlightWriter.c
@@ -149,8 +149,8 @@ HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) {
uint32_t freq;
UNUSED_VAR(self);
- // Leave space for a c32 indicating the number of postings.
- BB_Set_Size(tv_buf, C32_MAX_BYTES);
+ // Leave space for a cu32 indicating the number of postings.
+ BB_Set_Size(tv_buf, CU32_MAX_BYTES);
Inversion_Reset(inversion);
while ((tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL) {
@@ -164,11 +164,11 @@ HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) {
char *orig;
size_t old_size = BB_Get_Size(tv_buf);
size_t new_size = old_size
- + C32_MAX_BYTES // overlap
- + C32_MAX_BYTES // length of string diff
+ + CI32_MAX_BYTES // overlap
+ + CI32_MAX_BYTES // length of string diff
+ (token_len - overlap) // diff char data
- + C32_MAX_BYTES // num prox
- + (C32_MAX_BYTES * freq * 3); // pos data
+ + CU32_MAX_BYTES // num prox
+ + (CU32_MAX_BYTES * freq * 3); // pos data
// Allocate for worst-case scenario.
ptr = BB_Grow(tv_buf, new_size);
@@ -179,8 +179,8 @@ HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) {
num_postings += 1;
// Append the string diff to the tv_buf.
- NumUtil_encode_c32(overlap, &ptr);
- NumUtil_encode_c32((token_len - overlap), &ptr);
+ NumUtil_encode_ci32(overlap, &ptr);
+ NumUtil_encode_ci32((token_len - overlap), &ptr);
memcpy(ptr, (token_text + overlap), (token_len - overlap));
ptr += token_len - overlap;
@@ -189,13 +189,13 @@ HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) {
last_len = token_len;
// Append the number of positions for this term.
- NumUtil_encode_c32(freq, &ptr);
+ NumUtil_encode_cu32(freq, &ptr);
do {
// Add position, start_offset, and end_offset to tv_buf.
- NumUtil_encode_c32(Token_Get_Pos(token), &ptr);
- NumUtil_encode_c32(Token_Get_Start_Offset(token), &ptr);
- NumUtil_encode_c32(Token_Get_End_Offset(token), &ptr);
+ NumUtil_encode_ci32(Token_Get_Pos(token), &ptr);
+ NumUtil_encode_cu32(Token_Get_Start_Offset(token), &ptr);
+ NumUtil_encode_cu32(Token_Get_End_Offset(token), &ptr);
} while (--freq && (token = *++tokens));
// Set new byte length.
@@ -204,7 +204,7 @@ HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) {
// Go back and start the term vector string with the posting count.
char *dest = BB_Get_Buf(tv_buf);
- NumUtil_encode_padded_c32(num_postings, &dest);
+ NumUtil_encode_padded_cu32(num_postings, &dest);
Blob *blob = BB_Yield_Blob(tv_buf);
DECREF(tv_buf);
http://git-wip-us.apache.org/repos/asf/lucy/blob/67656d94/core/Lucy/Index/Posting/RichPosting.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Posting/RichPosting.c b/core/Lucy/Index/Posting/RichPosting.c
index 17e3e95..b3ff7e4 100644
--- a/core/Lucy/Index/Posting/RichPosting.c
+++ b/core/Lucy/Index/Posting/RichPosting.c
@@ -33,12 +33,12 @@
#include "Lucy/Util/MemoryPool.h"
#include "Lucy/Util/NumberUtils.h"
-#define FREQ_MAX_LEN C32_MAX_BYTES
+#define FREQ_MAX_LEN CU32_MAX_BYTES
#define MAX_RAW_POSTING_LEN(_raw_posting_size, _text_len, _freq) \
( _raw_posting_size \
+ _text_len /* term text content */ \
- + FREQ_MAX_LEN /* freq c32 */ \
- + (C32_MAX_BYTES * _freq) /* positions deltas */ \
+ + FREQ_MAX_LEN /* freq cu32 */ \
+ + (CU32_MAX_BYTES * _freq) /* positions deltas */ \
+ _freq /* per-pos boost byte */ \
)
@@ -137,7 +137,7 @@ RichPost_Add_Inversion_To_Pool_IMP(RichPosting *self, PostingPool *post_pool,
const uint32_t prox_delta = t_ivars->pos - last_prox;
const float boost = field_boost * t_ivars->boost;
- NumUtil_encode_c32(prox_delta, &dest);
+ NumUtil_encode_cu32(prox_delta, &dest);
last_prox = t_ivars->pos;
*((uint8_t*)dest) = Sim_Encode_Norm(sim, boost);
http://git-wip-us.apache.org/repos/asf/lucy/blob/67656d94/core/Lucy/Index/Posting/ScorePosting.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Posting/ScorePosting.c b/core/Lucy/Index/Posting/ScorePosting.c
index 3e995a8..bc84bf8 100644
--- a/core/Lucy/Index/Posting/ScorePosting.c
+++ b/core/Lucy/Index/Posting/ScorePosting.c
@@ -35,13 +35,13 @@
#include "Lucy/Util/NumberUtils.h"
#define FIELD_BOOST_LEN 1
-#define FREQ_MAX_LEN C32_MAX_BYTES
+#define FREQ_MAX_LEN CU32_MAX_BYTES
#define MAX_RAW_POSTING_LEN(_raw_post_size, _text_len, _freq) \
( _raw_post_size \
+ _text_len /* term text content */ \
+ FIELD_BOOST_LEN /* field boost byte */ \
- + FREQ_MAX_LEN /* freq c32 */ \
- + (C32_MAX_BYTES * _freq) /* positions deltas */ \
+ + FREQ_MAX_LEN /* freq cu32 */ \
+ + (CU32_MAX_BYTES * _freq) /* positions deltas */ \
)
ScorePosting*
@@ -110,7 +110,7 @@ ScorePost_Add_Inversion_To_Pool_IMP(ScorePosting *self,
for (uint32_t i = 0; i < freq; i++) {
TokenIVARS *const t_ivars = Token_IVARS(tokens[i]);
const uint32_t prox_delta = t_ivars->pos - last_prox;
- NumUtil_encode_c32(prox_delta, &dest);
+ NumUtil_encode_cu32(prox_delta, &dest);
last_prox = t_ivars->pos;
}
@@ -134,9 +134,9 @@ void
ScorePost_Read_Record_IMP(ScorePosting *self, InStream *instream) {
ScorePostingIVARS *const ivars = ScorePost_IVARS(self);
uint32_t position = 0;
- const size_t max_start_bytes = (C32_MAX_BYTES * 2) + 1;
+ const size_t max_start_bytes = (CU32_MAX_BYTES * 2) + 1;
const char *buf = InStream_Buf(instream, max_start_bytes);
- const uint32_t doc_code = NumUtil_decode_c32(&buf);
+ const uint32_t doc_code = NumUtil_decode_cu32(&buf);
const uint32_t doc_delta = doc_code >> 1;
// Apply delta doc and retrieve freq.
@@ -145,7 +145,7 @@ ScorePost_Read_Record_IMP(ScorePosting *self, InStream *instream) {
ivars->freq = 1;
}
else {
- ivars->freq = NumUtil_decode_c32(&buf);
+ ivars->freq = NumUtil_decode_cu32(&buf);
}
// Decode boost/norm byte.
@@ -162,9 +162,9 @@ ScorePost_Read_Record_IMP(ScorePosting *self, InStream *instream) {
uint32_t *positions = ivars->prox;
InStream_Advance_Buf(instream, buf);
- buf = InStream_Buf(instream, num_prox * C32_MAX_BYTES);
+ buf = InStream_Buf(instream, num_prox * CU32_MAX_BYTES);
while (num_prox--) {
- position += NumUtil_decode_c32(&buf);
+ position += NumUtil_decode_cu32(&buf);
*positions++ = position;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/67656d94/core/Lucy/Store/OutStream.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/OutStream.c b/core/Lucy/Store/OutStream.c
index 3f1d941..8ae07d4 100644
--- a/core/Lucy/Store/OutStream.c
+++ b/core/Lucy/Store/OutStream.c
@@ -306,7 +306,7 @@ OutStream_Write_CU32_IMP(OutStream *self, uint32_t value) {
static CFISH_INLINE void
SI_write_cu32(OutStream *self, OutStreamIVARS *ivars, uint32_t value) {
- uint8_t buf[C32_MAX_BYTES];
+ uint8_t buf[CU32_MAX_BYTES];
uint8_t *ptr = buf + sizeof(buf) - 1;
// Write last byte first, which has no continue bit.
@@ -339,7 +339,7 @@ OutStream_Write_CU64_IMP(OutStream *self, uint64_t value) {
static CFISH_INLINE void
SI_write_cu64(OutStream *self, OutStreamIVARS *ivars, uint64_t value) {
- uint8_t buf[C64_MAX_BYTES];
+ uint8_t buf[CU64_MAX_BYTES];
uint8_t *ptr = buf + sizeof(buf) - 1;
// Write last byte first, which has no continue bit.
[03/18] lucy git commit: Read/write compressed signed integers.
Posted by ma...@apache.org.
Read/write compressed signed integers.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/ce89b9c4
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/ce89b9c4
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/ce89b9c4
Branch: refs/heads/master
Commit: ce89b9c40ea86b45c181852a12edeaa7bea2d054
Parents: 3e3c648
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Mon Apr 18 15:43:38 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Mon Apr 18 15:43:38 2016 -0700
----------------------------------------------------------------------
core/Lucy/Store/InStream.c | 10 ++++
core/Lucy/Store/InStream.cfh | 11 ++++
core/Lucy/Store/OutStream.c | 10 ++++
core/Lucy/Store/OutStream.cfh | 10 ++++
core/Lucy/Test/Store/TestIOPrimitives.c | 82 +++++++++++++++++++++++++++-
5 files changed, 122 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/ce89b9c4/core/Lucy/Store/InStream.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/InStream.c b/core/Lucy/Store/InStream.c
index 09da76f..d04652d 100644
--- a/core/Lucy/Store/InStream.c
+++ b/core/Lucy/Store/InStream.c
@@ -477,6 +477,11 @@ InStream_Read_C32_IMP(InStream *self) {
return SI_read_cu32(self);
}
+int32_t
+InStream_Read_CI32_IMP(InStream *self) {
+ return (int32_t)SI_read_cu32(self);
+}
+
uint32_t
InStream_Read_CU32_IMP(InStream *self) {
return SI_read_cu32(self);
@@ -501,6 +506,11 @@ InStream_Read_C64_IMP(InStream *self) {
return SI_read_cu64(self);
}
+int64_t
+InStream_Read_CI64_IMP(InStream *self) {
+ return (int64_t)SI_read_cu64(self);
+}
+
uint64_t
InStream_Read_CU64_IMP(InStream *self) {
return SI_read_cu64(self);
http://git-wip-us.apache.org/repos/asf/lucy/blob/ce89b9c4/core/Lucy/Store/InStream.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/InStream.cfh b/core/Lucy/Store/InStream.cfh
index 5fd5dab..921be77 100644
--- a/core/Lucy/Store/InStream.cfh
+++ b/core/Lucy/Store/InStream.cfh
@@ -176,6 +176,11 @@ class Lucy::Store::InStream inherits Clownfish::Obj {
uint32_t
Read_C32(InStream *self);
+ /** Read in a compressed 32-bit signed integer.
+ */
+ int32_t
+ Read_CI32(InStream *self);
+
/** Read in a compressed 32-bit unsigned integer.
*/
uint32_t
@@ -187,6 +192,12 @@ class Lucy::Store::InStream inherits Clownfish::Obj {
final uint64_t
Read_C64(InStream *self);
+ /** Read a 64-bit signed integer, using the same encoding as a CI32 but
+ * occupying as many as 10 bytes.
+ */
+ final int64_t
+ Read_CI64(InStream *self);
+
/** Read a 64-bit unsigned integer, using the same encoding as a CU32 but
* occupying as many as 10 bytes.
*/
http://git-wip-us.apache.org/repos/asf/lucy/blob/ce89b9c4/core/Lucy/Store/OutStream.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/OutStream.c b/core/Lucy/Store/OutStream.c
index 8839791..3f1d941 100644
--- a/core/Lucy/Store/OutStream.c
+++ b/core/Lucy/Store/OutStream.c
@@ -295,6 +295,11 @@ OutStream_Write_C32_IMP(OutStream *self, uint32_t value) {
}
void
+OutStream_Write_CI32_IMP(OutStream *self, int32_t value) {
+ SI_write_cu32(self, OutStream_IVARS(self), (uint32_t)value);
+}
+
+void
OutStream_Write_CU32_IMP(OutStream *self, uint32_t value) {
SI_write_cu32(self, OutStream_IVARS(self), value);
}
@@ -323,6 +328,11 @@ OutStream_Write_C64_IMP(OutStream *self, uint64_t value) {
}
void
+OutStream_Write_CI64_IMP(OutStream *self, int64_t value) {
+ SI_write_cu64(self, OutStream_IVARS(self), (uint64_t)value);
+}
+
+void
OutStream_Write_CU64_IMP(OutStream *self, uint64_t value) {
SI_write_cu64(self, OutStream_IVARS(self), value);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/ce89b9c4/core/Lucy/Store/OutStream.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/OutStream.cfh b/core/Lucy/Store/OutStream.cfh
index 3726101..cb67003 100644
--- a/core/Lucy/Store/OutStream.cfh
+++ b/core/Lucy/Store/OutStream.cfh
@@ -118,6 +118,11 @@ class Lucy::Store::OutStream inherits Clownfish::Obj {
final void
Write_C32(OutStream *self, uint32_t value);
+ /** Write a signed 32-bit integer using a compressed format.
+ */
+ final void
+ Write_CI32(OutStream *self, int32_t value);
+
/** Write an unsigned 32-bit integer using a compressed format.
*/
final void
@@ -128,6 +133,11 @@ class Lucy::Store::OutStream inherits Clownfish::Obj {
final void
Write_C64(OutStream *self, uint64_t value);
+ /** Write a signed 64-bit integer using a compressed format.
+ */
+ final void
+ Write_CI64(OutStream *self, int64_t value);
+
/** Write an unsigned 64-bit integer using a compressed format.
*/
final void
http://git-wip-us.apache.org/repos/asf/lucy/blob/ce89b9c4/core/Lucy/Test/Store/TestIOPrimitives.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Test/Store/TestIOPrimitives.c b/core/Lucy/Test/Store/TestIOPrimitives.c
index 3c09dde..8d0f76f 100644
--- a/core/Lucy/Test/Store/TestIOPrimitives.c
+++ b/core/Lucy/Test/Store/TestIOPrimitives.c
@@ -242,6 +242,45 @@ test_u64(TestBatchRunner *runner) {
}
static void
+test_ci32(TestBatchRunner *runner) {
+ int64_t *ints = TestUtils_random_i64s(NULL, 1000, INT32_MIN, INT32_MAX);
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ InStream *instream;
+ size_t i;
+
+ // Test boundaries.
+ ints[0] = 0;
+ ints[1] = 1;
+ ints[2] = -1;
+ ints[3] = INT32_MAX;
+ ints[4] = INT32_MIN;
+
+ for (i = 0; i < 1000; i++) {
+ OutStream_Write_CI32(outstream, (int32_t)ints[i]);
+ }
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ for (i = 0; i < 1000; i++) {
+ int32_t got = InStream_Read_CI32(instream);
+ if ((int64_t)got != ints[i]) {
+ FAIL(runner, "ci32 round trip failed: %" PRId32 ", %" PRId64,
+ got, ints[i]);
+ break;
+ }
+ }
+ if (i == 1000) {
+ PASS(runner, "ci32 round trip");
+ }
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+ FREEMEM(ints);
+}
+
+static void
test_cu32(TestBatchRunner *runner) {
uint64_t *ints = TestUtils_random_u64s(NULL, 1000, 0, UINT32_MAX);
RAMFile *file = RAMFile_new(NULL, false);
@@ -280,6 +319,45 @@ test_cu32(TestBatchRunner *runner) {
}
static void
+test_ci64(TestBatchRunner *runner) {
+ int64_t *ints = TestUtils_random_i64s(NULL, 1000, INT64_MIN, INT64_MAX);
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ InStream *instream;
+ uint32_t i;
+
+ // Test boundaries.
+ ints[0] = 0;
+ ints[1] = 1;
+ ints[2] = -1;
+ ints[3] = INT64_MAX;
+ ints[4] = INT64_MIN;
+
+ for (i = 0; i < 1000; i++) {
+ OutStream_Write_CI64(outstream, ints[i]);
+ }
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ for (i = 0; i < 1000; i++) {
+ int64_t got = InStream_Read_CI64(instream);
+ if (got != ints[i]) {
+ FAIL(runner, "ci64 round trip failed: %" PRId64 ", %" PRId64,
+ got, ints[i]);
+ break;
+ }
+ }
+ if (i == 1000) {
+ PASS(runner, "ci64 round trip");
+ }
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+ FREEMEM(ints);
+}
+
+static void
test_cu64(TestBatchRunner *runner) {
uint64_t *ints = TestUtils_random_u64s(NULL, 1000, 0, UINT64_MAX);
RAMFile *file = RAMFile_new(NULL, false);
@@ -421,7 +499,7 @@ test_f64(TestBatchRunner *runner) {
void
TestIOPrimitives_Run_IMP(TestIOPrimitives *self, TestBatchRunner *runner) {
- TestBatchRunner_Plan(runner, (TestBatch*)self, 11);
+ TestBatchRunner_Plan(runner, (TestBatch*)self, 13);
srand((unsigned int)time((time_t*)NULL));
test_i8(runner);
test_u8(runner);
@@ -429,7 +507,9 @@ TestIOPrimitives_Run_IMP(TestIOPrimitives *self, TestBatchRunner *runner) {
test_u32(runner);
test_i64(runner);
test_u64(runner);
+ test_ci32(runner);
test_cu32(runner);
+ test_ci64(runner);
test_cu64(runner);
test_f32(runner);
test_f64(runner);
[12/18] lucy git commit: Remove obsolete C32/C64 code.
Posted by ma...@apache.org.
Remove obsolete C32/C64 code.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/ab5a44bd
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/ab5a44bd
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/ab5a44bd
Branch: refs/heads/master
Commit: ab5a44bd414e79ed0a5127e303887056ce09d7cd
Parents: febd2ff
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Wed Apr 20 19:23:29 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Wed Apr 20 19:26:48 2016 -0700
----------------------------------------------------------------------
core/Lucy/Store/InStream.c | 10 ----------
core/Lucy/Store/InStream.cfh | 11 -----------
core/Lucy/Store/OutStream.c | 10 ----------
core/Lucy/Store/OutStream.cfh | 10 ----------
go/build.go | 4 ----
go/lucy/store.go | 32 --------------------------------
6 files changed, 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/ab5a44bd/core/Lucy/Store/InStream.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/InStream.c b/core/Lucy/Store/InStream.c
index d04652d..ab78b0d 100644
--- a/core/Lucy/Store/InStream.c
+++ b/core/Lucy/Store/InStream.c
@@ -472,11 +472,6 @@ InStream_Read_F64_IMP(InStream *self) {
return duo.d;
}
-uint32_t
-InStream_Read_C32_IMP(InStream *self) {
- return SI_read_cu32(self);
-}
-
int32_t
InStream_Read_CI32_IMP(InStream *self) {
return (int32_t)SI_read_cu32(self);
@@ -501,11 +496,6 @@ SI_read_cu32(InStream *self) {
return retval;
}
-uint64_t
-InStream_Read_C64_IMP(InStream *self) {
- return SI_read_cu64(self);
-}
-
int64_t
InStream_Read_CI64_IMP(InStream *self) {
return (int64_t)SI_read_cu64(self);
http://git-wip-us.apache.org/repos/asf/lucy/blob/ab5a44bd/core/Lucy/Store/InStream.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/InStream.cfh b/core/Lucy/Store/InStream.cfh
index 921be77..3d96860 100644
--- a/core/Lucy/Store/InStream.cfh
+++ b/core/Lucy/Store/InStream.cfh
@@ -171,11 +171,6 @@ class Lucy::Store::InStream inherits Clownfish::Obj {
final double
Read_F64(InStream *self);
- /** Read in a compressed 32-bit unsigned integer.
- */
- uint32_t
- Read_C32(InStream *self);
-
/** Read in a compressed 32-bit signed integer.
*/
int32_t
@@ -186,12 +181,6 @@ class Lucy::Store::InStream inherits Clownfish::Obj {
uint32_t
Read_CU32(InStream *self);
- /** Read a 64-bit integer, using the same encoding as a C32 but occupying
- * as many as 10 bytes.
- */
- final uint64_t
- Read_C64(InStream *self);
-
/** Read a 64-bit signed integer, using the same encoding as a CI32 but
* occupying as many as 10 bytes.
*/
http://git-wip-us.apache.org/repos/asf/lucy/blob/ab5a44bd/core/Lucy/Store/OutStream.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/OutStream.c b/core/Lucy/Store/OutStream.c
index 8ae07d4..7e619a4 100644
--- a/core/Lucy/Store/OutStream.c
+++ b/core/Lucy/Store/OutStream.c
@@ -290,11 +290,6 @@ OutStream_Write_F64_IMP(OutStream *self, double value) {
}
void
-OutStream_Write_C32_IMP(OutStream *self, uint32_t value) {
- SI_write_cu32(self, OutStream_IVARS(self), value);
-}
-
-void
OutStream_Write_CI32_IMP(OutStream *self, int32_t value) {
SI_write_cu32(self, OutStream_IVARS(self), (uint32_t)value);
}
@@ -323,11 +318,6 @@ SI_write_cu32(OutStream *self, OutStreamIVARS *ivars, uint32_t value) {
}
void
-OutStream_Write_C64_IMP(OutStream *self, uint64_t value) {
- SI_write_cu64(self, OutStream_IVARS(self), value);
-}
-
-void
OutStream_Write_CI64_IMP(OutStream *self, int64_t value) {
SI_write_cu64(self, OutStream_IVARS(self), (uint64_t)value);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/ab5a44bd/core/Lucy/Store/OutStream.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/OutStream.cfh b/core/Lucy/Store/OutStream.cfh
index cb67003..391418d 100644
--- a/core/Lucy/Store/OutStream.cfh
+++ b/core/Lucy/Store/OutStream.cfh
@@ -113,11 +113,6 @@ class Lucy::Store::OutStream inherits Clownfish::Obj {
final void
Write_U64(OutStream *self, uint64_t value);
- /** Write a 32-bit integer using a compressed format.
- */
- final void
- Write_C32(OutStream *self, uint32_t value);
-
/** Write a signed 32-bit integer using a compressed format.
*/
final void
@@ -128,11 +123,6 @@ class Lucy::Store::OutStream inherits Clownfish::Obj {
final void
Write_CU32(OutStream *self, uint32_t value);
- /** Write a 64-bit integer using a compressed format.
- */
- final void
- Write_C64(OutStream *self, uint64_t value);
-
/** Write a signed 64-bit integer using a compressed format.
*/
final void
http://git-wip-us.apache.org/repos/asf/lucy/blob/ab5a44bd/go/build.go
----------------------------------------------------------------------
diff --git a/go/build.go b/go/build.go
index ecf36f4..3fea7a0 100644
--- a/go/build.go
+++ b/go/build.go
@@ -387,10 +387,8 @@ func specClasses(parcel *cfc.Parcel) {
inStreamBinding.SpecMethod("Read_U8", "ReadU8() (uint8, error)")
inStreamBinding.SpecMethod("Read_U32", "ReadU32() (uint32, error)")
inStreamBinding.SpecMethod("Read_U64", "ReadU64() (uint64, error)")
- inStreamBinding.SpecMethod("Read_C32", "ReadC32() (uint32, error)")
inStreamBinding.SpecMethod("Read_CI32", "ReadCI32() (int32, error)")
inStreamBinding.SpecMethod("Read_CU32", "ReadCU32() (uint32, error)")
- inStreamBinding.SpecMethod("Read_C64", "ReadC64() (uint64, error)")
inStreamBinding.SpecMethod("Read_CI64", "ReadCI64() (int64, error)")
inStreamBinding.SpecMethod("Read_CU64", "ReadCU64() (uint64, error)")
inStreamBinding.SpecMethod("Read_F32", "ReadF32() (float32, error)")
@@ -409,10 +407,8 @@ func specClasses(parcel *cfc.Parcel) {
outStreamBinding.SpecMethod("Write_U8", "WriteU8(uint8) error")
outStreamBinding.SpecMethod("Write_U32", "WriteU32(uint32) error")
outStreamBinding.SpecMethod("Write_U64", "WriteU64(uint64) error")
- outStreamBinding.SpecMethod("Write_C32", "WriteC32(uint32) error")
outStreamBinding.SpecMethod("Write_CI32", "WriteCI32(int32) error")
outStreamBinding.SpecMethod("Write_CU32", "WriteCU32(uint32) error")
- outStreamBinding.SpecMethod("Write_C64", "WriteC64(uint64) error")
outStreamBinding.SpecMethod("Write_CI64", "WriteCI64(int64) error")
outStreamBinding.SpecMethod("Write_CU64", "WriteCU64(uint64) error")
outStreamBinding.SpecMethod("Write_F32", "WriteF32(float32) error")
http://git-wip-us.apache.org/repos/asf/lucy/blob/ab5a44bd/go/lucy/store.go
----------------------------------------------------------------------
diff --git a/go/lucy/store.go b/go/lucy/store.go
index 0b11a2f..a543ece 100644
--- a/go/lucy/store.go
+++ b/go/lucy/store.go
@@ -171,15 +171,6 @@ func (in *InStreamIMP) ReadU64() (uint64, error) {
return retval, err
}
-func (in *InStreamIMP) ReadC32() (uint32, error) {
- var retval uint32
- err := clownfish.TrapErr(func() {
- self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in"))
- retval = uint32(C.LUCY_InStream_Read_C32(self))
- })
- return retval, err
-}
-
func (in *InStreamIMP) ReadCI32() (int32, error) {
var retval int32
err := clownfish.TrapErr(func() {
@@ -198,15 +189,6 @@ func (in *InStreamIMP) ReadCU32() (uint32, error) {
return retval, err
}
-func (in *InStreamIMP) ReadC64() (uint64, error) {
- var retval uint64
- err := clownfish.TrapErr(func() {
- self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in"))
- retval = uint64(C.LUCY_InStream_Read_C64(self))
- })
- return retval, err
-}
-
func (in *InStreamIMP) ReadCI64() (int64, error) {
var retval int64
err := clownfish.TrapErr(func() {
@@ -343,13 +325,6 @@ func (out *OutStreamIMP) WriteU64(value uint64) error {
})
}
-func (out *OutStreamIMP) WriteC32(value uint32) error {
- return clownfish.TrapErr(func() {
- self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out"))
- C.LUCY_OutStream_Write_C32(self, C.uint32_t(value))
- })
-}
-
func (out *OutStreamIMP) WriteCI32(value int32) error {
return clownfish.TrapErr(func() {
self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out"))
@@ -364,13 +339,6 @@ func (out *OutStreamIMP) WriteCU32(value uint32) error {
})
}
-func (out *OutStreamIMP) WriteC64(value uint64) error {
- return clownfish.TrapErr(func() {
- self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out"))
- C.LUCY_OutStream_Write_C64(self, C.uint64_t(value))
- })
-}
-
func (out *OutStreamIMP) WriteCI64(value int64) error {
return clownfish.TrapErr(func() {
self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out"))
[09/18] lucy git commit: Change easy-to-review Read_C32 to CU32.
Posted by ma...@apache.org.
Change easy-to-review Read_C32 to CU32.
All of these have the type of the lvalue show up in the diff, making it
easy to confirm that the types match.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/17a3d60b
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/17a3d60b
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/17a3d60b
Branch: refs/heads/master
Commit: 17a3d60bbe2b5a1279052278225b2875668b2968
Parents: 795626b
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Wed Apr 20 15:29:23 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Wed Apr 20 15:29:23 2016 -0700
----------------------------------------------------------------------
c/src/Lucy/Index/DocReader.c | 8 ++++----
core/Lucy/Index/HighlightReader.c | 2 +-
core/Lucy/Index/LexIndex.c | 4 ++--
core/Lucy/Index/Posting/MatchPosting.c | 6 +++---
core/Lucy/Index/Posting/RichPosting.c | 6 +++---
core/Lucy/Index/Posting/ScorePosting.c | 4 ++--
core/Lucy/Plan/TextType.c | 6 +++---
core/Lucy/Search/SortSpec.c | 2 +-
core/Lucy/Util/Freezer.c | 12 ++++++------
core/LucyX/Search/ProximityQuery.c | 2 +-
10 files changed, 26 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/17a3d60b/c/src/Lucy/Index/DocReader.c
----------------------------------------------------------------------
diff --git a/c/src/Lucy/Index/DocReader.c b/c/src/Lucy/Index/DocReader.c
index 1e7767b..3610dfc 100644
--- a/c/src/Lucy/Index/DocReader.c
+++ b/c/src/Lucy/Index/DocReader.c
@@ -71,7 +71,7 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) {
// Read the field value.
switch (FType_Primitive_ID(type) & FType_PRIMITIVE_ID_MASK) {
case FType_TEXT: {
- uint32_t value_len = InStream_Read_C32(dat_in);
+ uint32_t value_len = InStream_Read_CU32(dat_in);
char *buf = (char*)MALLOCATE(value_len + 1);
InStream_Read_Bytes(dat_in, buf, value_len);
buf[value_len] = '\0';
@@ -79,7 +79,7 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) {
break;
}
case FType_BLOB: {
- uint32_t value_len = InStream_Read_C32(dat_in);
+ uint32_t value_len = InStream_Read_CU32(dat_in);
char *buf = (char*)MALLOCATE(value_len);
InStream_Read_Bytes(dat_in, buf, value_len);
value = (Obj*)Blob_new_steal(buf, value_len);
@@ -92,10 +92,10 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) {
value = (Obj*)Float_new(InStream_Read_F64(dat_in));
break;
case FType_INT32:
- value = (Obj*)Int_new((int32_t)InStream_Read_C32(dat_in));
+ value = (Obj*)Int_new(InStream_Read_CI32(dat_in));
break;
case FType_INT64:
- value = (Obj*)Int_new((int64_t)InStream_Read_C64(dat_in));
+ value = (Obj*)Int_new(InStream_Read_CI64(dat_in));
break;
default:
value = NULL;
http://git-wip-us.apache.org/repos/asf/lucy/blob/17a3d60b/core/Lucy/Index/HighlightReader.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/HighlightReader.c b/core/Lucy/Index/HighlightReader.c
index 2afaf66..590a6bd 100644
--- a/core/Lucy/Index/HighlightReader.c
+++ b/core/Lucy/Index/HighlightReader.c
@@ -200,7 +200,7 @@ DefHLReader_Fetch_Doc_Vec_IMP(DefaultHighlightReader *self, int32_t doc_id) {
int64_t file_pos = InStream_Read_I64(ix_in);
InStream_Seek(dat_in, file_pos);
- uint32_t num_fields = InStream_Read_C32(dat_in);
+ uint32_t num_fields = InStream_Read_CU32(dat_in);
while (num_fields--) {
String *field = Freezer_read_string(dat_in);
Blob *field_buf = Freezer_read_blob(dat_in);
http://git-wip-us.apache.org/repos/asf/lucy/blob/17a3d60b/core/Lucy/Index/LexIndex.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/LexIndex.c b/core/Lucy/Index/LexIndex.c
index 162933f..2c6586b 100644
--- a/core/Lucy/Index/LexIndex.c
+++ b/core/Lucy/Index/LexIndex.c
@@ -129,11 +129,11 @@ S_read_entry(LexIndex *self) {
int64_t offset = (int64_t)NumUtil_decode_bigend_u64(ivars->offsets + ivars->tick);
InStream_Seek(ix_in, offset);
TermStepper_Read_Key_Frame(ivars->term_stepper, ix_in);
- int32_t doc_freq = InStream_Read_C32(ix_in);
+ int32_t doc_freq = InStream_Read_CI32(ix_in);
TInfo_Set_Doc_Freq(tinfo, doc_freq);
TInfo_Set_Post_FilePos(tinfo, InStream_Read_C64(ix_in));
int64_t skip_filepos = doc_freq >= ivars->skip_interval
- ? InStream_Read_C64(ix_in)
+ ? InStream_Read_CI64(ix_in)
: 0;
TInfo_Set_Skip_FilePos(tinfo, skip_filepos);
TInfo_Set_Lex_FilePos(tinfo, InStream_Read_C64(ix_in));
http://git-wip-us.apache.org/repos/asf/lucy/blob/17a3d60b/core/Lucy/Index/Posting/MatchPosting.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Posting/MatchPosting.c b/core/Lucy/Index/Posting/MatchPosting.c
index c0ead4e..ae4e1f2 100644
--- a/core/Lucy/Index/Posting/MatchPosting.c
+++ b/core/Lucy/Index/Posting/MatchPosting.c
@@ -81,7 +81,7 @@ MatchPost_Reset_IMP(MatchPosting *self) {
void
MatchPost_Read_Record_IMP(MatchPosting *self, InStream *instream) {
MatchPostingIVARS *const ivars = MatchPost_IVARS(self);
- const uint32_t doc_code = InStream_Read_C32(instream);
+ const uint32_t doc_code = InStream_Read_CU32(instream);
const uint32_t doc_delta = doc_code >> 1;
// Apply delta doc and retrieve freq.
@@ -100,12 +100,12 @@ MatchPost_Read_Raw_IMP(MatchPosting *self, InStream *instream,
MemoryPool *mem_pool) {
const char *const text_buf = Str_Get_Ptr8(term_text);
const size_t text_size = Str_Get_Size(term_text);
- const uint32_t doc_code = InStream_Read_C32(instream);
+ const uint32_t doc_code = InStream_Read_CU32(instream);
const uint32_t delta_doc = doc_code >> 1;
const int32_t doc_id = last_doc_id + delta_doc;
const uint32_t freq = (doc_code & 1)
? 1
- : InStream_Read_C32(instream);
+ : InStream_Read_CU32(instream);
const size_t base_size = Class_Get_Obj_Alloc_Size(RAWPOSTING);
size_t raw_post_bytes = MAX_RAW_POSTING_LEN(base_size, text_size);
void *const allocation = MemPool_Grab(mem_pool, raw_post_bytes);
http://git-wip-us.apache.org/repos/asf/lucy/blob/17a3d60b/core/Lucy/Index/Posting/RichPosting.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Posting/RichPosting.c b/core/Lucy/Index/Posting/RichPosting.c
index b3ff7e4..db590d4 100644
--- a/core/Lucy/Index/Posting/RichPosting.c
+++ b/core/Lucy/Index/Posting/RichPosting.c
@@ -72,7 +72,7 @@ RichPost_Read_Record_IMP(RichPosting *self, InStream *instream) {
float aggregate_weight = 0.0;
// Decode delta doc.
- uint32_t doc_code = InStream_Read_C32(instream);
+ uint32_t doc_code = InStream_Read_CU32(instream);
ivars->doc_id += doc_code >> 1;
// If the stored num was odd, the freq is 1.
@@ -158,12 +158,12 @@ RichPost_Read_Raw_IMP(RichPosting *self, InStream *instream,
MemoryPool *mem_pool) {
const char *const text_buf = Str_Get_Ptr8(term_text);
const size_t text_size = Str_Get_Size(term_text);
- const uint32_t doc_code = InStream_Read_C32(instream);
+ const uint32_t doc_code = InStream_Read_CU32(instream);
const uint32_t delta_doc = doc_code >> 1;
const int32_t doc_id = last_doc_id + delta_doc;
const uint32_t freq = (doc_code & 1)
? 1
- : InStream_Read_C32(instream);
+ : InStream_Read_CU32(instream);
const size_t base_size = Class_Get_Obj_Alloc_Size(RAWPOSTING);
size_t raw_post_bytes = MAX_RAW_POSTING_LEN(base_size, text_size, freq);
void *const allocation = MemPool_Grab(mem_pool, raw_post_bytes);
http://git-wip-us.apache.org/repos/asf/lucy/blob/17a3d60b/core/Lucy/Index/Posting/ScorePosting.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/Posting/ScorePosting.c b/core/Lucy/Index/Posting/ScorePosting.c
index bc84bf8..048fa21 100644
--- a/core/Lucy/Index/Posting/ScorePosting.c
+++ b/core/Lucy/Index/Posting/ScorePosting.c
@@ -177,12 +177,12 @@ ScorePost_Read_Raw_IMP(ScorePosting *self, InStream *instream,
MemoryPool *mem_pool) {
const char *const text_buf = Str_Get_Ptr8(term_text);
const size_t text_size = Str_Get_Size(term_text);
- const uint32_t doc_code = InStream_Read_C32(instream);
+ const uint32_t doc_code = InStream_Read_CU32(instream);
const uint32_t delta_doc = doc_code >> 1;
const int32_t doc_id = last_doc_id + delta_doc;
const uint32_t freq = (doc_code & 1)
? 1
- : InStream_Read_C32(instream);
+ : InStream_Read_CU32(instream);
const size_t base_size = Class_Get_Obj_Alloc_Size(RAWPOSTING);
size_t raw_post_bytes = MAX_RAW_POSTING_LEN(base_size, text_size, freq);
void *const allocation = MemPool_Grab(mem_pool, raw_post_bytes);
http://git-wip-us.apache.org/repos/asf/lucy/blob/17a3d60b/core/Lucy/Plan/TextType.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Plan/TextType.c b/core/Lucy/Plan/TextType.c
index 4737b8c..4b84fa8 100644
--- a/core/Lucy/Plan/TextType.c
+++ b/core/Lucy/Plan/TextType.c
@@ -151,7 +151,7 @@ void
TextTermStepper_Read_Key_Frame_IMP(TextTermStepper *self,
InStream *instream) {
TextTermStepperIVARS *const ivars = TextTermStepper_IVARS(self);
- const uint32_t text_len = InStream_Read_C32(instream);
+ const uint32_t text_len = InStream_Read_CU32(instream);
// Allocate space.
char *ptr = BB_Grow(ivars->bytebuf, text_len);
@@ -173,8 +173,8 @@ TextTermStepper_Read_Key_Frame_IMP(TextTermStepper *self,
void
TextTermStepper_Read_Delta_IMP(TextTermStepper *self, InStream *instream) {
TextTermStepperIVARS *const ivars = TextTermStepper_IVARS(self);
- const uint32_t text_overlap = InStream_Read_C32(instream);
- const uint32_t finish_chars_len = InStream_Read_C32(instream);
+ const uint32_t text_overlap = InStream_Read_CU32(instream);
+ const uint32_t finish_chars_len = InStream_Read_CU32(instream);
const uint32_t total_text_len = text_overlap + finish_chars_len;
// Allocate space.
http://git-wip-us.apache.org/repos/asf/lucy/blob/17a3d60b/core/Lucy/Search/SortSpec.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Search/SortSpec.c b/core/Lucy/Search/SortSpec.c
index c6fa2e4..9994bb4 100644
--- a/core/Lucy/Search/SortSpec.c
+++ b/core/Lucy/Search/SortSpec.c
@@ -53,7 +53,7 @@ SortSpec_Destroy_IMP(SortSpec *self) {
SortSpec*
SortSpec_Deserialize_IMP(SortSpec *self, InStream *instream) {
- uint32_t num_rules = InStream_Read_C32(instream);
+ uint32_t num_rules = InStream_Read_CU32(instream);
Vector *rules = Vec_new(num_rules);
// Add rules.
http://git-wip-us.apache.org/repos/asf/lucy/blob/17a3d60b/core/Lucy/Util/Freezer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/Freezer.c b/core/Lucy/Util/Freezer.c
index fa0d121..20e5e33 100644
--- a/core/Lucy/Util/Freezer.c
+++ b/core/Lucy/Util/Freezer.c
@@ -126,7 +126,7 @@ Freezer_deserialize(Obj *obj, InStream *instream) {
obj = (Obj*)Freezer_deserialize_hash((Hash*)obj, instream);
}
else if (Obj_is_a(obj, INTEGER)) {
- int64_t value = (int64_t)InStream_Read_C64(instream);
+ int64_t value = InStream_Read_CI64(instream);
obj = (Obj*)Int_init((Integer*)obj, value);
}
else if (Obj_is_a(obj, FLOAT)) {
@@ -245,11 +245,11 @@ Freezer_serialize_varray(Vector *array, OutStream *outstream) {
Vector*
Freezer_deserialize_varray(Vector *array, InStream *instream) {
- uint32_t size = InStream_Read_C32(instream);
+ uint32_t size = InStream_Read_CU32(instream);
Vec_init(array, size);
- for (uint32_t tick = InStream_Read_C32(instream);
+ for (uint32_t tick = InStream_Read_CU32(instream);
tick < size;
- tick += InStream_Read_C32(instream)
+ tick += InStream_Read_CU32(instream)
) {
Obj *obj = THAW(instream);
Vec_Store(array, tick, obj);
@@ -281,12 +281,12 @@ Freezer_serialize_hash(Hash *hash, OutStream *outstream) {
Hash*
Freezer_deserialize_hash(Hash *hash, InStream *instream) {
- uint32_t size = InStream_Read_C32(instream);
+ uint32_t size = InStream_Read_CU32(instream);
Hash_init(hash, size);
while (size--) {
- uint32_t len = InStream_Read_C32(instream);
+ uint32_t len = InStream_Read_CU32(instream);
char *key_buf = (char*)MALLOCATE(len + 1);
InStream_Read_Bytes(instream, key_buf, len);
key_buf[len] = '\0';
http://git-wip-us.apache.org/repos/asf/lucy/blob/17a3d60b/core/LucyX/Search/ProximityQuery.c
----------------------------------------------------------------------
diff --git a/core/LucyX/Search/ProximityQuery.c b/core/LucyX/Search/ProximityQuery.c
index 6b01d5b..637fcce 100644
--- a/core/LucyX/Search/ProximityQuery.c
+++ b/core/LucyX/Search/ProximityQuery.c
@@ -95,7 +95,7 @@ ProximityQuery_Deserialize_IMP(ProximityQuery *self, InStream *instream) {
float boost = InStream_Read_F32(instream);
String *field = Freezer_read_string(instream);
Vector *terms = Freezer_read_varray(instream);
- uint32_t within = InStream_Read_C32(instream);
+ uint32_t within = InStream_Read_CU32(instream);
return S_do_init(self, field, terms, boost, within);
}
[06/18] lucy git commit: Remove obsolete NumUtil C32/C64 code.
Posted by ma...@apache.org.
Remove obsolete NumUtil C32/C64 code.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/c1af0933
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/c1af0933
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/c1af0933
Branch: refs/heads/master
Commit: c1af09331739ee47e4a9dc138b7dd0265ba74497
Parents: 67656d9
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Mon Apr 18 18:45:19 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Mon Apr 18 18:45:19 2016 -0700
----------------------------------------------------------------------
core/Lucy/Util/NumberUtils.cfh | 73 -------------------------------------
1 file changed, 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/c1af0933/core/Lucy/Util/NumberUtils.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/NumberUtils.cfh b/core/Lucy/Util/NumberUtils.cfh
index 8d1b0b1..7928906 100644
--- a/core/Lucy/Util/NumberUtils.cfh
+++ b/core/Lucy/Util/NumberUtils.cfh
@@ -80,13 +80,6 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline double
decode_bigend_f64(const void *source);
- /** Encode a C32 at the space pointed to by `dest`. As a side
- * effect, `dest` will be advanced to immediately after the end
- * of the C32.
- */
- inert inline void
- encode_c32(uint32_t value, char **dest);
-
/** Encode a compressed 32-bit signed integer at the space pointed to by
* `dest`. As a side effect, `dest` will be advanced to immediately after
* the end of the compressed data.
@@ -101,14 +94,6 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline void
encode_cu32(uint32_t value, char **dest);
- /** Encode a C32 at the space pointed to by `dest`, but add
- * "leading zeroes" so that the space consumed will always be 5 bytes. As
- * a side effect, `dest` will be advanced to immediately after
- * the end of the C32.
- */
- inert inline void
- encode_padded_c32(uint32_t value, char **dest);
-
/** Encode a compressed 32-bit unsigned integer at the space pointed to by
* `dest`, but add "leading zeroes" so that the space consumed will always
* be 5 bytes. As a side effect, `dest` will be advanced to immediately
@@ -117,13 +102,6 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline void
encode_padded_cu32(uint32_t value, char **dest);
- /** Encode a C64 at the space pointed to by `dest`. As a side
- * effect, `dest` will be advanced to immediately after the end
- * of the C64.
- */
- inert inline void
- encode_c64(uint64_t value, char **dest);
-
/** Encode a compressed 64-bit signed integer at the space pointed to by
* `dest`. As a side effect, `dest` will be advanced to immediately after
* the end of the compressed data.
@@ -138,13 +116,6 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline void
encode_cu64(uint64_t value, char **dest);
- /** Read a C32 from the buffer pointed to by `source`. As a
- * side effect, advance the pointer, consuming the bytes occupied by the
- * C32.
- */
- inert inline uint32_t
- decode_c32(const char **source);
-
/** Read a compressed 32-bit signed integer from the buffer pointed to
* by `source`. As a side effect, advance the pointer, consuming the
* bytes occupied by the compressed number.
@@ -159,13 +130,6 @@ inert class Lucy::Util::NumberUtils nickname NumUtil {
inert inline uint32_t
decode_cu32(const char **source);
- /** Read a C64 from the buffer pointed to by `source`. As a
- * side effect, advance the pointer, consuming the bytes occupied by the
- * C64.
- */
- inert inline uint64_t
- decode_c64(const char **source);
-
/** Read a compressed 64-bit signed integer from the buffer pointed to
* by `source`. As a side effect, advance the pointer, consuming the
* bytes occupied by the compressed number.
@@ -358,19 +322,12 @@ lucy_NumUtil_decode_bigend_f64(const void *source) {
return duo.d;
}
-#define LUCY_NUMUTIL_C32_MAX_BYTES ((sizeof(uint32_t) * 8 / 7) + 1) /* 5 */
-#define LUCY_NUMUTIL_C64_MAX_BYTES ((sizeof(uint64_t) * 8 / 7) + 1) /* 10 */
#define LUCY_NUMUTIL_CI32_MAX_BYTES ((sizeof(int32_t) * 8 / 7) + 1) /* 5 */
#define LUCY_NUMUTIL_CU32_MAX_BYTES ((sizeof(uint32_t) * 8 / 7) + 1) /* 5 */
#define LUCY_NUMUTIL_CI64_MAX_BYTES ((sizeof(int64_t) * 8 / 7) + 1) /* 10 */
#define LUCY_NUMUTIL_CU64_MAX_BYTES ((sizeof(uint64_t) * 8 / 7) + 1) /* 10 */
static CFISH_INLINE void
-lucy_NumUtil_encode_c32(uint32_t value, char **out_buf) {
- lucy_NumUtil_encode_cu32(value, out_buf);
-}
-
-static CFISH_INLINE void
lucy_NumUtil_encode_ci32(int32_t value, char **out_buf) {
lucy_NumUtil_encode_cu32((uint32_t)value, out_buf);
}
@@ -395,11 +352,6 @@ lucy_NumUtil_encode_cu32(uint32_t value, char **out_buf) {
}
static CFISH_INLINE void
-lucy_NumUtil_encode_c64(uint64_t value, char **out_buf) {
- lucy_NumUtil_encode_cu64(value, out_buf);
-}
-
-static CFISH_INLINE void
lucy_NumUtil_encode_ci64(int64_t value, char **out_buf) {
lucy_NumUtil_encode_cu64((uint64_t)value, out_buf);
}
@@ -424,11 +376,6 @@ lucy_NumUtil_encode_cu64(uint64_t value, char **out_buf) {
}
static CFISH_INLINE void
-lucy_NumUtil_encode_padded_c32(uint32_t value, char **out_buf) {
- lucy_NumUtil_encode_padded_cu32(value, out_buf);
-}
-
-static CFISH_INLINE void
lucy_NumUtil_encode_padded_cu32(uint32_t value, char **out_buf) {
uint8_t buf[LUCY_NUMUTIL_CU32_MAX_BYTES]
= { 0x80, 0x80, 0x80, 0x80, 0x80 };
@@ -455,15 +402,6 @@ lucy_NumUtil_encode_padded_cu32(uint32_t value, char **out_buf) {
} \
} while (0)
-static CFISH_INLINE uint32_t
-lucy_NumUtil_decode_c32(const char **source_ptr) {
- const char *source = *source_ptr;
- uint32_t decoded;
- LUCY_NUMUTIL_DECODE_CINT(decoded, source);
- *source_ptr = source;
- return decoded;
-}
-
static CFISH_INLINE int32_t
lucy_NumUtil_decode_ci32(const char **source_ptr) {
return (int32_t)lucy_NumUtil_decode_cu32(source_ptr);
@@ -478,15 +416,6 @@ lucy_NumUtil_decode_cu32(const char **source_ptr) {
return decoded;
}
-static CFISH_INLINE uint64_t
-lucy_NumUtil_decode_c64(const char **source_ptr) {
- const char *source = *source_ptr;
- uint64_t decoded;
- LUCY_NUMUTIL_DECODE_CINT(decoded, source);
- *source_ptr = source;
- return decoded;
-}
-
static CFISH_INLINE int64_t
lucy_NumUtil_decode_ci64(const char **source_ptr) {
return (int64_t)lucy_NumUtil_decode_cu64(source_ptr);
@@ -578,8 +507,6 @@ lucy_NumUtil_u4set(void *array, uint32_t tick, uint8_t value) {
}
#ifdef LUCY_USE_SHORT_NAMES
- #define C32_MAX_BYTES LUCY_NUMUTIL_C32_MAX_BYTES
- #define C64_MAX_BYTES LUCY_NUMUTIL_C64_MAX_BYTES
#define CI32_MAX_BYTES LUCY_NUMUTIL_CI32_MAX_BYTES
#define CI64_MAX_BYTES LUCY_NUMUTIL_CI64_MAX_BYTES
#define CU32_MAX_BYTES LUCY_NUMUTIL_CU32_MAX_BYTES
[07/18] lucy git commit: Switch Perl bindings from C32/C64 to
CI32/etc.
Posted by ma...@apache.org.
Switch Perl bindings from C32/C64 to CI32/etc.
Change IO methods invoked to use explicitly signed/unsigned versions
rather than casting.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/cbe964df
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/cbe964df
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/cbe964df
Branch: refs/heads/master
Commit: cbe964dfa85720357b8e6f06347028671c641e06
Parents: c1af093
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Tue Apr 19 18:59:15 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Tue Apr 19 19:20:16 2016 -0700
----------------------------------------------------------------------
perl/buildlib/Lucy/Build/Binding/Store.pm | 7 +++++--
perl/lib/LucyX/Search/Filter.pm | 4 ++--
perl/t/023-stepper.t | 4 ++--
perl/t/102-strings_io.t | 2 +-
perl/t/binding/101-simple_io.t | 20 ++++++++++----------
perl/xs/Lucy/Document/Doc.c | 8 ++++----
perl/xs/Lucy/Index/DocReader.c | 14 +++++++-------
7 files changed, 31 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/cbe964df/perl/buildlib/Lucy/Build/Binding/Store.pm
----------------------------------------------------------------------
diff --git a/perl/buildlib/Lucy/Build/Binding/Store.pm b/perl/buildlib/Lucy/Build/Binding/Store.pm
index a266ace..f5196c8 100644
--- a/perl/buildlib/Lucy/Build/Binding/Store.pm
+++ b/perl/buildlib/Lucy/Build/Binding/Store.pm
@@ -175,7 +175,7 @@ read_string(self)
CODE:
{
char *ptr;
- size_t len = LUCY_InStream_Read_C32(self);
+ size_t len = LUCY_InStream_Read_CU32(self);
RETVAL = newSV(len + 1);
SvCUR_set(RETVAL, len);
SvPOK_on(RETVAL);
@@ -364,7 +364,10 @@ PPCODE:
{
STRLEN len = 0;
char *ptr = SvPVutf8(aSV, len);
- LUCY_OutStream_Write_C32(self, len);
+ if (len > INT32_MAX) {
+ CFISH_THROW(CFISH_ERR, "String too long: %u64", (uint64_t)len);
+ }
+ LUCY_OutStream_Write_CU32(self, len);
LUCY_OutStream_Write_Bytes(self, ptr, len);
}
END_XS_CODE
http://git-wip-us.apache.org/repos/asf/lucy/blob/cbe964df/perl/lib/LucyX/Search/Filter.pm
----------------------------------------------------------------------
diff --git a/perl/lib/LucyX/Search/Filter.pm b/perl/lib/LucyX/Search/Filter.pm
index d168e9e..d6db91f 100644
--- a/perl/lib/LucyX/Search/Filter.pm
+++ b/perl/lib/LucyX/Search/Filter.pm
@@ -63,14 +63,14 @@ sub serialize {
my ( $self, $outstream ) = @_;
$self->SUPER::serialize($outstream);
my $frozen = nfreeze( $query{$$self} );
- $outstream->write_c32( bytes::length($frozen) );
+ $outstream->write_cu32( bytes::length($frozen) );
$outstream->print($frozen);
}
sub deserialize {
my ( $self, $instream ) = @_;
$self->SUPER::deserialize($instream);
- my $len = $instream->read_c32;
+ my $len = $instream->read_cu32;
my $frozen;
$instream->read( $frozen, $len );
$query{$$self} = thaw($frozen);
http://git-wip-us.apache.org/repos/asf/lucy/blob/cbe964df/perl/t/023-stepper.t
----------------------------------------------------------------------
diff --git a/perl/t/023-stepper.t b/perl/t/023-stepper.t
index 7ea2fa8..2786ad7 100644
--- a/perl/t/023-stepper.t
+++ b/perl/t/023-stepper.t
@@ -38,7 +38,7 @@ sub get_number { $number{ ${ +shift } } }
sub read_record {
my ( $self, $instream ) = @_;
- $number{$$self} += $instream->read_c32;
+ $number{$$self} += $instream->read_ci32;
}
package main;
@@ -47,7 +47,7 @@ use Lucy::Test;
my $folder = Lucy::Store::RAMFolder->new;
my $outstream = $folder->open_out("foo") or die Clownfish->error;
-$outstream->write_c32(10) for 1 .. 5;
+$outstream->write_ci32(10) for 1 .. 5;
$outstream->close;
my $instream = $folder->open_in("foo") or die Clownfish->error;
my $stepper = MyStepper->new;
http://git-wip-us.apache.org/repos/asf/lucy/blob/cbe964df/perl/t/102-strings_io.t
----------------------------------------------------------------------
diff --git a/perl/t/102-strings_io.t b/perl/t/102-strings_io.t
index 657bf46..02d7ca8 100644
--- a/perl/t/102-strings_io.t
+++ b/perl/t/102-strings_io.t
@@ -24,7 +24,7 @@ my ( @items, $packed, $template, $buf, $file, $out, $in, $correct );
$file = Lucy::Store::RAMFile->new;
$out = Lucy::Store::OutStream->open( file => $file )
or die Clownfish->error;
-$out->write_c64(10000);
+$out->write_cu64(10000);
$out->close;
$in = Lucy::Store::InStream->open( file => $file )
or die Clownfish->error;
http://git-wip-us.apache.org/repos/asf/lucy/blob/cbe964df/perl/t/binding/101-simple_io.t
----------------------------------------------------------------------
diff --git a/perl/t/binding/101-simple_io.t b/perl/t/binding/101-simple_io.t
index fd5a9be..85db57a 100644
--- a/perl/t/binding/101-simple_io.t
+++ b/perl/t/binding/101-simple_io.t
@@ -47,7 +47,7 @@ sub check_round_trip_bytes {
my $file = Lucy::Store::RAMFile->new;
my $outstream = Lucy::Store::OutStream->open( file => $file );
for (@$expected) {
- $outstream->write_c32( bytes::length($_) );
+ $outstream->write_ci32( bytes::length($_) );
$outstream->print($_);
}
$outstream->close;
@@ -56,7 +56,7 @@ sub check_round_trip_bytes {
my @got;
for (@$expected) {
my $buf;
- my $len = $instream->read_c32;
+ my $len = $instream->read_ci32;
$instream->read( $buf, $len );
push @got, $buf;
}
@@ -95,24 +95,24 @@ $_ += int( rand( 2**16 ) ) for @nums;
check_round_trip( 'u64', \@nums );
@nums = ( 0 .. 127 );
-check_round_trip( 'c32', \@nums );
+check_round_trip( 'ci32', \@nums );
@nums = ( 128 .. 500 );
$packed = pack( 'w*', @nums );
-$ram_file = check_round_trip( 'c32', \@nums );
-is( $ram_file->get_contents, $packed, "C32 is equivalent to Perl's pack w" );
+$ram_file = check_round_trip( 'cu32', \@nums );
+is( $ram_file->get_contents, $packed, "CU32 is equivalent to Perl's pack w" );
@nums = ( 0 .. 127 );
-check_round_trip( 'c64', \@nums );
+check_round_trip( 'ci64', \@nums );
@nums = ( 128 .. 500 );
$packed = pack( 'w*', @nums );
-$ram_file = check_round_trip( 'c64', \@nums );
-is( $ram_file->get_contents, $packed, "C64 is equivalent to Perl's pack w" );
+$ram_file = check_round_trip( 'cu64', \@nums );
+is( $ram_file->get_contents, $packed, "CU64 is equivalent to Perl's pack w" );
@nums = map { $_ * 2**31 } 0 .. 2000;
$_ += int( rand( 2**16 ) ) for @nums;
-check_round_trip( 'c64', \@nums );
+check_round_trip( 'cu64', \@nums );
# rand (always?) has 64-bit precision, but we need 32-bit - so truncate via
# pack/unpack.
@@ -152,6 +152,6 @@ my $unibytes = $latin;
utf8ify($unibytes);
utf8_flag_off($unibytes);
my $slurped = $ram_file->get_contents;
-substr( $slurped, 0, 1, "" ); # ditch c32 at head of string;
+substr( $slurped, 0, 1, "" ); # ditch cu32 at head of string;
is( $slurped, $unibytes, "write_string upgrades to utf8" );
http://git-wip-us.apache.org/repos/asf/lucy/blob/cbe964df/perl/xs/Lucy/Document/Doc.c
----------------------------------------------------------------------
diff --git a/perl/xs/Lucy/Document/Doc.c b/perl/xs/Lucy/Document/Doc.c
index 902df23..a923e25 100644
--- a/perl/xs/Lucy/Document/Doc.c
+++ b/perl/xs/Lucy/Document/Doc.c
@@ -92,11 +92,11 @@ void
LUCY_Doc_Serialize_IMP(lucy_Doc *self, lucy_OutStream *outstream) {
dTHX;
lucy_DocIVARS *const ivars = lucy_Doc_IVARS(self);
- LUCY_OutStream_Write_C32(outstream, ivars->doc_id);
+ LUCY_OutStream_Write_CU32(outstream, ivars->doc_id);
SV *frozen = S_nfreeze_fields(aTHX_ self);
STRLEN len;
char *buf = SvPV(frozen, len);
- LUCY_OutStream_Write_C64(outstream, len);
+ LUCY_OutStream_Write_CU64(outstream, len);
LUCY_OutStream_Write_Bytes(outstream, buf, len);
SvREFCNT_dec(frozen);
}
@@ -104,7 +104,7 @@ LUCY_Doc_Serialize_IMP(lucy_Doc *self, lucy_OutStream *outstream) {
static HV*
S_thaw_fields(pTHX_ lucy_InStream *instream) {
// Read frozen data into an SV buffer.
- size_t len = (size_t)LUCY_InStream_Read_C64(instream);
+ size_t len = (size_t)LUCY_InStream_Read_CU64(instream);
SV *buf_sv = newSV(len + 1);
SvPOK_on(buf_sv);
SvCUR_set(buf_sv, len);
@@ -137,7 +137,7 @@ S_thaw_fields(pTHX_ lucy_InStream *instream) {
lucy_Doc*
LUCY_Doc_Deserialize_IMP(lucy_Doc *self, lucy_InStream *instream) {
dTHX;
- int32_t doc_id = (int32_t)LUCY_InStream_Read_C32(instream);
+ int32_t doc_id = (int32_t)LUCY_InStream_Read_CU32(instream);
HV *fields = S_thaw_fields(aTHX_ instream);
lucy_Doc_init(self, fields, doc_id);
SvREFCNT_dec(fields);
http://git-wip-us.apache.org/repos/asf/lucy/blob/cbe964df/perl/xs/Lucy/Index/DocReader.c
----------------------------------------------------------------------
diff --git a/perl/xs/Lucy/Index/DocReader.c b/perl/xs/Lucy/Index/DocReader.c
index 8eda8bc..7daf909 100644
--- a/perl/xs/Lucy/Index/DocReader.c
+++ b/perl/xs/Lucy/Index/DocReader.c
@@ -43,7 +43,7 @@ LUCY_DefDocReader_Fetch_Doc_IMP(lucy_DefaultDocReader *self, int32_t doc_id) {
LUCY_InStream_Seek(ix_in, (int64_t)doc_id * 8);
start = LUCY_InStream_Read_U64(ix_in);
LUCY_InStream_Seek(dat_in, start);
- num_fields = LUCY_InStream_Read_C32(dat_in);
+ num_fields = LUCY_InStream_Read_CU32(dat_in);
// Decode stored data and build up the doc field by field.
while (num_fields--) {
@@ -53,7 +53,7 @@ LUCY_DefDocReader_Fetch_Doc_IMP(lucy_DefaultDocReader *self, int32_t doc_id) {
lucy_FieldType *type;
// Read field name.
- field_name_len = LUCY_InStream_Read_C32(dat_in);
+ field_name_len = LUCY_InStream_Read_CU32(dat_in);
field_name_ptr = SvGROW(field_name_sv, field_name_len + 1);
LUCY_InStream_Read_Bytes(dat_in, field_name_ptr, field_name_len);
SvPOK_on(field_name_sv);
@@ -69,7 +69,7 @@ LUCY_DefDocReader_Fetch_Doc_IMP(lucy_DefaultDocReader *self, int32_t doc_id) {
// Read the field value.
switch (LUCY_FType_Primitive_ID(type) & lucy_FType_PRIMITIVE_ID_MASK) {
case lucy_FType_TEXT: {
- STRLEN value_len = LUCY_InStream_Read_C32(dat_in);
+ STRLEN value_len = LUCY_InStream_Read_CU32(dat_in);
value_sv = newSV((value_len ? value_len : 1));
LUCY_InStream_Read_Bytes(dat_in, SvPVX(value_sv), value_len);
SvCUR_set(value_sv, value_len);
@@ -79,7 +79,7 @@ LUCY_DefDocReader_Fetch_Doc_IMP(lucy_DefaultDocReader *self, int32_t doc_id) {
break;
}
case lucy_FType_BLOB: {
- STRLEN value_len = LUCY_InStream_Read_C32(dat_in);
+ STRLEN value_len = LUCY_InStream_Read_CU32(dat_in);
value_sv = newSV((value_len ? value_len : 1));
LUCY_InStream_Read_Bytes(dat_in, SvPVX(value_sv), value_len);
SvCUR_set(value_sv, value_len);
@@ -94,15 +94,15 @@ LUCY_DefDocReader_Fetch_Doc_IMP(lucy_DefaultDocReader *self, int32_t doc_id) {
value_sv = newSVnv(LUCY_InStream_Read_F64(dat_in));
break;
case lucy_FType_INT32:
- value_sv = newSViv((int32_t)LUCY_InStream_Read_C32(dat_in));
+ value_sv = newSViv(LUCY_InStream_Read_CI32(dat_in));
break;
case lucy_FType_INT64:
if (sizeof(IV) == 8) {
- int64_t val = (int64_t)LUCY_InStream_Read_C64(dat_in);
+ int64_t val = LUCY_InStream_Read_CI64(dat_in);
value_sv = newSViv((IV)val);
}
else { // (lossy)
- int64_t val = (int64_t)LUCY_InStream_Read_C64(dat_in);
+ int64_t val = LUCY_InStream_Read_CI64(dat_in);
value_sv = newSVnv((double)val);
}
break;
[17/18] lucy git commit: Avoid PRId32 for the sake of Windows.
Posted by ma...@apache.org.
Avoid PRId32 for the sake of Windows.
For the time being, Charmonizer knows about PRId64 but not PRId32.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/6c167aa5
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/6c167aa5
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/6c167aa5
Branch: refs/heads/master
Commit: 6c167aa5acc6f2a680184c7bd823500fbce1f0eb
Parents: 7876b13
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Thu Apr 21 14:02:32 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Thu Apr 21 14:02:32 2016 -0700
----------------------------------------------------------------------
core/Lucy/Test/Store/TestIOPrimitives.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/6c167aa5/core/Lucy/Test/Store/TestIOPrimitives.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Test/Store/TestIOPrimitives.c b/core/Lucy/Test/Store/TestIOPrimitives.c
index 8d0f76f..9b78e82 100644
--- a/core/Lucy/Test/Store/TestIOPrimitives.c
+++ b/core/Lucy/Test/Store/TestIOPrimitives.c
@@ -265,8 +265,8 @@ test_ci32(TestBatchRunner *runner) {
for (i = 0; i < 1000; i++) {
int32_t got = InStream_Read_CI32(instream);
if ((int64_t)got != ints[i]) {
- FAIL(runner, "ci32 round trip failed: %" PRId32 ", %" PRId64,
- got, ints[i]);
+ FAIL(runner, "ci32 round trip failed: %" PRId64 ", %" PRId64,
+ (int64_t)got, ints[i]);
break;
}
}