You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by ma...@apache.org on 2014/07/03 03:09:12 UTC
[4/5] git commit: refs/heads/264_dont_subclass_hash - Eliminate
MemoryPool from SortFieldWriter.
Eliminate MemoryPool from SortFieldWriter.
SortFieldWriter used to allocate values from the MemoryPool, which was
also used to track memory usage. Now it uses ordinary objects and
memory consumption is tracked in a simple Counter, so the MemoryPool is
no longer needed.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/d0910e43
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/d0910e43
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/d0910e43
Branch: refs/heads/264_dont_subclass_hash
Commit: d0910e438b24b5cc9c8e04781cea96ed29f94d3d
Parents: 2f32088
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Wed Jul 2 17:50:46 2014 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Wed Jul 2 18:08:44 2014 -0700
----------------------------------------------------------------------
core/Lucy/Index/SortFieldWriter.c | 22 ++++++++--------------
core/Lucy/Index/SortFieldWriter.cfh | 5 ++---
core/Lucy/Index/SortWriter.c | 9 +++------
core/Lucy/Index/SortWriter.cfh | 1 -
4 files changed, 13 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/d0910e43/core/Lucy/Index/SortFieldWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortFieldWriter.c b/core/Lucy/Index/SortFieldWriter.c
index 40c896a..fc350ba 100644
--- a/core/Lucy/Index/SortFieldWriter.c
+++ b/core/Lucy/Index/SortFieldWriter.c
@@ -36,7 +36,6 @@
#include "Lucy/Store/InStream.h"
#include "Lucy/Store/OutStream.h"
#include "Clownfish/Util/Memory.h"
-#include "Lucy/Util/MemoryPool.h"
#include "Clownfish/Util/SortUtils.h"
// Prepare to read back a run.
@@ -67,13 +66,13 @@ SI_increase_to_word_multiple(int64_t amount) {
SortFieldWriter*
SortFieldWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
PolyReader *polyreader, String *field,
- MemoryPool *memory_pool, Counter *counter, size_t mem_thresh,
+ Counter *counter, size_t mem_thresh,
OutStream *temp_ord_out, OutStream *temp_ix_out,
OutStream *temp_dat_out) {
SortFieldWriter *self
= (SortFieldWriter*)VTable_Make_Obj(SORTFIELDWRITER);
return SortFieldWriter_init(self, schema, snapshot, segment, polyreader,
- field, memory_pool, counter, mem_thresh, temp_ord_out,
+ field, counter, mem_thresh, temp_ord_out,
temp_ix_out, temp_dat_out);
}
@@ -81,7 +80,7 @@ SortFieldWriter*
SortFieldWriter_init(SortFieldWriter *self, Schema *schema,
Snapshot *snapshot, Segment *segment,
PolyReader *polyreader, String *field,
- MemoryPool *memory_pool, Counter *counter, size_t mem_thresh,
+ Counter *counter, size_t mem_thresh,
OutStream *temp_ord_out, OutStream *temp_ix_out,
OutStream *temp_dat_out) {
// Init.
@@ -109,7 +108,6 @@ SortFieldWriter_init(SortFieldWriter *self, Schema *schema,
ivars->snapshot = (Snapshot*)INCREF(snapshot);
ivars->segment = (Segment*)INCREF(segment);
ivars->polyreader = (PolyReader*)INCREF(polyreader);
- ivars->mem_pool = (MemoryPool*)INCREF(memory_pool);
ivars->counter = (Counter*)INCREF(counter);
ivars->temp_ord_out = (OutStream*)INCREF(temp_ord_out);
ivars->temp_ix_out = (OutStream*)INCREF(temp_ix_out);
@@ -142,10 +140,10 @@ SortFieldWriter_Clear_Buffer_IMP(SortFieldWriter *self) {
SortFieldWriter_Clear_Buffer_t super_clear_buffer
= SUPER_METHOD_PTR(SORTFIELDWRITER, LUCY_SortFieldWriter_Clear_Buffer);
super_clear_buffer(self);
- // Note that we have not called MemPool_Release_All() on our memory pool.
- // This is because the pool is shared amongst multiple SortFieldWriters
+ // Note that we have not Reset() the Counter which tracks memory usage.
+ // This is because the counter is shared amongst multiple SortFieldWriters
// which belong to a parent SortWriter; it is the responsibility of the
- // parent SortWriter to release the memory pool once **all** of its child
+ // parent SortWriter to reset it once **all** of its child
// SortFieldWriters have cleared their buffers.
}
@@ -160,7 +158,6 @@ SortFieldWriter_Destroy_IMP(SortFieldWriter *self) {
DECREF(ivars->segment);
DECREF(ivars->polyreader);
DECREF(ivars->type);
- DECREF(ivars->mem_pool);
DECREF(ivars->counter);
DECREF(ivars->temp_ord_out);
DECREF(ivars->temp_ix_out);
@@ -220,7 +217,7 @@ SortFieldWriter_Add_Segment_IMP(SortFieldWriter *self, SegReader *reader,
SortFieldWriterIVARS *const ivars = SortFieldWriter_IVARS(self);
SortFieldWriter *run
= SortFieldWriter_new(ivars->schema, ivars->snapshot, ivars->segment,
- ivars->polyreader, ivars->field, ivars->mem_pool, ivars->counter,
+ ivars->polyreader, ivars->field, ivars->counter,
ivars->mem_thresh, NULL, NULL, NULL);
SortFieldWriterIVARS *const run_ivars = SortFieldWriter_IVARS(run);
run_ivars->sort_cache = (SortCache*)INCREF(sort_cache);
@@ -411,7 +408,7 @@ SortFieldWriter_Flush_IMP(SortFieldWriter *self) {
SortFieldWriter_Sort_Buffer(self);
SortFieldWriter *run
= SortFieldWriter_new(ivars->schema, ivars->snapshot, ivars->segment,
- ivars->polyreader, ivars->field, ivars->mem_pool, ivars->counter,
+ ivars->polyreader, ivars->field, ivars->counter,
ivars->mem_thresh, NULL, NULL, NULL);
SortFieldWriterIVARS *const run_ivars = SortFieldWriter_IVARS(run);
@@ -464,7 +461,6 @@ SortFieldWriter_Refill_IMP(SortFieldWriter *self) {
buf_count);
}
SortFieldWriter_Clear_Buffer(self);
- MemPool_Release_All(ivars->mem_pool);
Counter_Reset(ivars->counter);
S_lazy_init_sorted_ids(self);
@@ -683,9 +679,7 @@ S_flip_run(SortFieldWriter *run, size_t sub_thresh, InStream *ord_in,
run_ivars->flipped = true;
// Get our own slice of mem_thresh.
- DECREF(run_ivars->mem_pool);
DECREF(run_ivars->counter);
- run_ivars->mem_pool = MemPool_new(0);
run_ivars->counter = Counter_new();
run_ivars->mem_thresh = sub_thresh;
http://git-wip-us.apache.org/repos/asf/lucy/blob/d0910e43/core/Lucy/Index/SortFieldWriter.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortFieldWriter.cfh b/core/Lucy/Index/SortFieldWriter.cfh
index df82067..e2b996f 100644
--- a/core/Lucy/Index/SortFieldWriter.cfh
+++ b/core/Lucy/Index/SortFieldWriter.cfh
@@ -26,7 +26,6 @@ class Lucy::Index::SortFieldWriter
PolyReader *polyreader;
FieldType *type;
I32Array *doc_map;
- MemoryPool *mem_pool;
Counter *counter;
int32_t field_num;
int32_t null_ord;
@@ -55,14 +54,14 @@ class Lucy::Index::SortFieldWriter
inert incremented SortFieldWriter*
new(Schema *schema, Snapshot *snapshot, Segment *segment,
- PolyReader *polyreader, String *field, MemoryPool *memory_pool, Counter *counter,
+ PolyReader *polyreader, String *field, Counter *counter,
size_t mem_thresh, OutStream *temp_ord_out, OutStream *temp_ix_out,
OutStream *temp_dat_out);
inert SortFieldWriter*
init(SortFieldWriter *self, Schema *schema, Snapshot *snapshot,
Segment *segment, PolyReader *polyreader, String *field,
- MemoryPool *memory_pool, Counter *counter, size_t mem_thresh, OutStream *temp_ord_out,
+ Counter *counter, size_t mem_thresh, OutStream *temp_ord_out,
OutStream *temp_ix_out, OutStream *temp_dat_out);
void
http://git-wip-us.apache.org/repos/asf/lucy/blob/d0910e43/core/Lucy/Index/SortWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortWriter.c b/core/Lucy/Index/SortWriter.c
index fc323d0..bd3b690 100644
--- a/core/Lucy/Index/SortWriter.c
+++ b/core/Lucy/Index/SortWriter.c
@@ -34,7 +34,6 @@
#include "Lucy/Store/InStream.h"
#include "Lucy/Store/OutStream.h"
#include "Clownfish/Util/Memory.h"
-#include "Lucy/Util/MemoryPool.h"
#include "Clownfish/Util/SortUtils.h"
int32_t SortWriter_current_file_format = 3;
@@ -63,7 +62,6 @@ SortWriter_init(SortWriter *self, Schema *schema, Snapshot *snapshot,
ivars->temp_ord_out = NULL;
ivars->temp_ix_out = NULL;
ivars->temp_dat_out = NULL;
- ivars->mem_pool = MemPool_new(0);
ivars->counter = Counter_new();
ivars->mem_thresh = default_mem_thresh;
ivars->flush_at_finish = false;
@@ -81,7 +79,6 @@ SortWriter_Destroy_IMP(SortWriter *self) {
DECREF(ivars->temp_ord_out);
DECREF(ivars->temp_ix_out);
DECREF(ivars->temp_dat_out);
- DECREF(ivars->mem_pool);
DECREF(ivars->counter);
SUPER_DESTROY(self, SORTWRITER);
}
@@ -126,7 +123,7 @@ S_lazy_init_field_writer(SortWriter *self, int32_t field_num) {
String *field = Seg_Field_Name(ivars->segment, field_num);
field_writer
= SortFieldWriter_new(ivars->schema, ivars->snapshot, ivars->segment,
- ivars->polyreader, field, ivars->mem_pool, ivars->counter,
+ ivars->polyreader, field, ivars->counter,
ivars->mem_thresh, ivars->temp_ord_out,
ivars->temp_ix_out, ivars->temp_dat_out);
VA_Store(ivars->field_writers, field_num, (Obj*)field_writer);
@@ -152,14 +149,14 @@ SortWriter_Add_Inverted_Doc_IMP(SortWriter *self, Inverter *inverter,
}
// If our SortFieldWriters have collectively passed the memory threshold,
- // flush all of them, then release all unique values with a single action.
+ // flush all of them, then reset the counter which tracks memory
+ // consumption.
if (Counter_Get_Value(ivars->counter) > ivars->mem_thresh) {
for (uint32_t i = 0; i < VA_Get_Size(ivars->field_writers); i++) {
SortFieldWriter *const field_writer
= (SortFieldWriter*)VA_Fetch(ivars->field_writers, i);
if (field_writer) { SortFieldWriter_Flush(field_writer); }
}
- MemPool_Release_All(ivars->mem_pool);
Counter_Reset(ivars->counter);
ivars->flush_at_finish = true;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/d0910e43/core/Lucy/Index/SortWriter.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortWriter.cfh b/core/Lucy/Index/SortWriter.cfh
index ba9bd35..6f4a242 100644
--- a/core/Lucy/Index/SortWriter.cfh
+++ b/core/Lucy/Index/SortWriter.cfh
@@ -35,7 +35,6 @@ class Lucy::Index::SortWriter inherits Lucy::Index::DataWriter {
OutStream *temp_ord_out;
OutStream *temp_ix_out;
OutStream *temp_dat_out;
- MemoryPool *mem_pool;
Counter *counter;
size_t mem_thresh;
bool flush_at_finish;