You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by ma...@apache.org on 2016/04/21 23:13:51 UTC
[15/18] lucy git commit: Change size_t C32/C64 read/writes.
Change size_t C32/C64 read/writes.
Adapt Read_C32/Write_C32/Read_C64/Write_C64 to Read_CU32/etc for
variables representing memory sizes: String, Blob, Vector and Hash sizes
in particular.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/5e4dcd2e
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/5e4dcd2e
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/5e4dcd2e
Branch: refs/heads/master
Commit: 5e4dcd2e31a80ed20e86ca931bbbc565f6781820
Parents: 17a3d60
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Wed Apr 20 18:16:06 2016 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Wed Apr 20 19:26:48 2016 -0700
----------------------------------------------------------------------
c/src/Lucy/Index/DocReader.c | 6 +++---
core/Lucy/Index/DocWriter.c | 12 ++++++++++--
core/Lucy/Index/TermVector.c | 4 ++--
core/Lucy/Plan/TextType.c | 3 ++-
core/Lucy/Search/SortSpec.c | 3 ++-
core/Lucy/Util/Freezer.c | 22 +++++++++++++++-------
6 files changed, 34 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/c/src/Lucy/Index/DocReader.c
----------------------------------------------------------------------
diff --git a/c/src/Lucy/Index/DocReader.c b/c/src/Lucy/Index/DocReader.c
index 3610dfc..16e184e 100644
--- a/c/src/Lucy/Index/DocReader.c
+++ b/c/src/Lucy/Index/DocReader.c
@@ -40,7 +40,7 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) {
Hash *const fields = Hash_new(1);
int64_t start;
uint32_t num_fields;
- uint32_t field_name_cap = 31;
+ size_t field_name_cap = 31;
char *field_name = (char*)MALLOCATE(field_name_cap + 1);
// Get data file pointer from index, read number of fields.
@@ -51,12 +51,12 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) {
// Decode stored data and build up the doc field by field.
while (num_fields--) {
- uint32_t field_name_len;
+ size_t field_name_len;
Obj *value;
FieldType *type;
// Read field name.
- field_name_len = InStream_Read_C32(dat_in);
+ field_name_len = InStream_Read_CU32(dat_in);
if (field_name_len > field_name_cap) {
field_name_cap = field_name_len;
field_name = (char*)REALLOCATE(field_name,
http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Index/DocWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/DocWriter.c b/core/Lucy/Index/DocWriter.c
index 818f45b..ec583cc 100644
--- a/core/Lucy/Index/DocWriter.c
+++ b/core/Lucy/Index/DocWriter.c
@@ -119,14 +119,22 @@ DocWriter_Add_Inverted_Doc_IMP(DocWriter *self, Inverter *inverter,
case FType_TEXT: {
const char *buf = Str_Get_Ptr8((String*)value);
size_t size = Str_Get_Size((String*)value);
- OutStream_Write_C32(dat_out, size);
+ if (size > INT32_MAX) {
+ THROW(ERR, "Field %o over 2GB: %u64", field,
+ (uint64_t)size);
+ }
+ OutStream_Write_CU32(dat_out, (uint32_t)size);
OutStream_Write_Bytes(dat_out, buf, size);
break;
}
case FType_BLOB: {
const char *buf = Blob_Get_Buf((Blob*)value);
size_t size = Blob_Get_Size((Blob*)value);
- OutStream_Write_C32(dat_out, size);
+ if (size > INT32_MAX) {
+ THROW(ERR, "Field %o over 2GB: %u64", field,
+ (uint64_t)size);
+ }
+ OutStream_Write_CU32(dat_out, (uint32_t)size);
OutStream_Write_Bytes(dat_out, buf, size);
break;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Index/TermVector.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/TermVector.c b/core/Lucy/Index/TermVector.c
index b1a3391..9c3a6f9 100644
--- a/core/Lucy/Index/TermVector.c
+++ b/core/Lucy/Index/TermVector.c
@@ -90,7 +90,7 @@ TV_Serialize_IMP(TermVector *self, OutStream *target) {
Freezer_serialize_string(ivars->field, target);
Freezer_serialize_string(ivars->text, target);
- OutStream_Write_C64(target, ivars->num_pos);
+ OutStream_Write_CU64(target, ivars->num_pos);
for (size_t i = 0; i < ivars->num_pos; i++) {
OutStream_Write_C32(target, posits[i]);
@@ -103,7 +103,7 @@ TermVector*
TV_Deserialize_IMP(TermVector *self, InStream *instream) {
String *field = Freezer_read_string(instream);
String *text = Freezer_read_string(instream);
- size_t num_pos = InStream_Read_C64(instream);
+ size_t num_pos = InStream_Read_CU64(instream);
// Read positional data.
int32_t *posits = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Plan/TextType.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Plan/TextType.c b/core/Lucy/Plan/TextType.c
index 4b84fa8..67f64a3 100644
--- a/core/Lucy/Plan/TextType.c
+++ b/core/Lucy/Plan/TextType.c
@@ -101,7 +101,8 @@ TextTermStepper_Write_Key_Frame_IMP(TextTermStepper *self,
String *string = (String*)CERTIFY(value, STRING);
const char *buf = Str_Get_Ptr8(string);
size_t size = Str_Get_Size(string);
- OutStream_Write_C32(outstream, size);
+ // Skip size check because we're deep in the internals.
+ OutStream_Write_CU32(outstream, (uint32_t)size);
OutStream_Write_Bytes(outstream, buf, size);
S_set_value(self, value);
http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Search/SortSpec.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Search/SortSpec.c b/core/Lucy/Search/SortSpec.c
index 9994bb4..21a6370 100644
--- a/core/Lucy/Search/SortSpec.c
+++ b/core/Lucy/Search/SortSpec.c
@@ -75,8 +75,9 @@ SortSpec_Get_Rules_IMP(SortSpec *self) {
void
SortSpec_Serialize_IMP(SortSpec *self, OutStream *target) {
SortSpecIVARS *const ivars = SortSpec_IVARS(self);
+ // Skip size check.
uint32_t num_rules = (uint32_t)Vec_Get_Size(ivars->rules);
- OutStream_Write_C32(target, num_rules);
+ OutStream_Write_CU32(target, num_rules);
for (uint32_t i = 0; i < num_rules; i++) {
SortRule *rule = (SortRule*)Vec_Fetch(ivars->rules, i);
SortRule_Serialize(rule, target);
http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Util/Freezer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/Freezer.c b/core/Lucy/Util/Freezer.c
index 20e5e33..90a63c3 100644
--- a/core/Lucy/Util/Freezer.c
+++ b/core/Lucy/Util/Freezer.c
@@ -180,13 +180,16 @@ void
Freezer_serialize_string(String *string, OutStream *outstream) {
size_t size = Str_Get_Size(string);
const char *buf = Str_Get_Ptr8(string);
- OutStream_Write_C64(outstream, size);
+ if (size > INT32_MAX) {
+ THROW(ERR, "Can't serialize string above 2GB: %u64", (uint64_t)size);
+ }
+ OutStream_Write_CU64(outstream, size);
OutStream_Write_Bytes(outstream, buf, size);
}
String*
Freezer_deserialize_string(String *string, InStream *instream) {
- size_t size = InStream_Read_C32(instream);
+ size_t size = InStream_Read_CU32(instream);
if (size == SIZE_MAX) {
THROW(ERR, "Can't deserialize SIZE_MAX bytes");
}
@@ -208,13 +211,16 @@ Freezer_read_string(InStream *instream) {
void
Freezer_serialize_blob(Blob *blob, OutStream *outstream) {
size_t size = Blob_Get_Size(blob);
- OutStream_Write_C32(outstream, size);
+ if (size > INT32_MAX) {
+ THROW(ERR, "Can't serialize blob above 2GB: %u64", (uint64_t)size);
+ }
+ OutStream_Write_CU64(outstream, size);
OutStream_Write_Bytes(outstream, Blob_Get_Buf(blob), size);
}
Blob*
Freezer_deserialize_blob(Blob *blob, InStream *instream) {
- size_t size = InStream_Read_C32(instream);
+ size_t size = InStream_Read_CU32(instream);
char *buf = (char*)MALLOCATE(size);
InStream_Read_Bytes(instream, buf, size);
return Blob_init_steal(blob, buf, size);
@@ -229,8 +235,9 @@ Freezer_read_blob(InStream *instream) {
void
Freezer_serialize_varray(Vector *array, OutStream *outstream) {
uint32_t last_valid_tick = 0;
+ // Skip size check.
uint32_t size = (uint32_t)Vec_Get_Size(array);
- OutStream_Write_C32(outstream, size);
+ OutStream_Write_CU32(outstream, size);
for (uint32_t i = 0; i < size; i++) {
Obj *elem = Vec_Fetch(array, i);
if (elem) {
@@ -266,8 +273,9 @@ Freezer_read_varray(InStream *instream) {
void
Freezer_serialize_hash(Hash *hash, OutStream *outstream) {
- uint32_t hash_size = Hash_Get_Size(hash);
- OutStream_Write_C32(outstream, hash_size);
+ // Skip size check.
+ uint32_t hash_size = (uint32_t)Hash_Get_Size(hash);
+ OutStream_Write_CU32(outstream, hash_size);
HashIterator *iter = HashIter_new(hash);
while (HashIter_Next(iter)) {