You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by ma...@apache.org on 2014/07/03 22:37:34 UTC
[1/4] git commit: refs/heads/264_dont_subclass_hash - Remove test for
VTable.
Repository: lucy
Updated Branches:
refs/heads/264_dont_subclass_hash d136b8675 -> 4195da771
Remove test for VTable.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/09336bbb
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/09336bbb
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/09336bbb
Branch: refs/heads/264_dont_subclass_hash
Commit: 09336bbbe05b6ca3186285c0e00f6301b80b2ced
Parents: d136b86
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Wed Jul 2 19:28:56 2014 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Wed Jul 2 19:28:56 2014 -0700
----------------------------------------------------------------------
perl/t/021-vtable.t | 95 ------------------------------------------------
1 file changed, 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/09336bbb/perl/t/021-vtable.t
----------------------------------------------------------------------
diff --git a/perl/t/021-vtable.t b/perl/t/021-vtable.t
deleted file mode 100644
index 5141e7f..0000000
--- a/perl/t/021-vtable.t
+++ /dev/null
@@ -1,95 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-use strict;
-use warnings;
-
-package MyHash;
-use base qw( Clownfish::Hash );
-
-sub oodle { }
-
-package RAMFolderOfDeath;
-use base qw( Lucy::Store::RAMFolder );
-
-sub open_in {
- my ( $self, $filename ) = @_;
- die "Sweet, sweet death.";
-}
-
-package OnceRemoved;
-use base qw( Lucy::Search::Query );
-
-our $serialize_was_called = 0;
-sub serialize {
- my ( $self, $outstream ) = @_;
- $serialize_was_called++;
- $self->SUPER::serialize($outstream);
-}
-
-package TwiceRemoved;
-use base qw( OnceRemoved );
-
-package main;
-
-use Lucy::Test;
-use Test::More tests => 9;
-use Storable qw( nfreeze );
-
-{
- my $twice_removed = TwiceRemoved->new;
- # This triggers a call to Obj_Serialize() via the VTable dispatch.
- my $frozen = nfreeze($twice_removed);
- ok( $serialize_was_called,
- "Overridden method in intermediate class recognized" );
- my $vtable = $twice_removed->get_vtable;
- is( $vtable->get_name, "TwiceRemoved", "correct class" );
- my $parent_vtable = $vtable->get_parent;
- is( $parent_vtable->get_name, "OnceRemoved", "correct parent class" )
-}
-
-my $stringified;
-my $storage = Clownfish::Hash->new;
-
-{
- my $subclassed_hash = MyHash->new;
- $stringified = $subclassed_hash->to_string;
-
- isa_ok( $subclassed_hash, "MyHash", "Perl isa reports correct subclass" );
-
- # Store the subclassed object. At the end of this block, the Perl object
- # will go out of scope and DESTROY will be called, but the Clownfish object
- # will persist.
- $storage->store( "test", $subclassed_hash );
-}
-
-my $resurrected = $storage->_fetch("test");
-
-isa_ok( $resurrected, "MyHash", "subclass name survived Perl destruction" );
-is( $resurrected->to_string, $stringified,
- "It's the same Hash from earlier (though a different Perl object)" );
-
-my $booga = Clownfish::String->new("booga");
-$resurrected->store( "ooga", $booga );
-
-is( $resurrected->fetch("ooga"),
- "booga", "subclassed object still performs correctly at the C level" );
-
-my $methods = Clownfish::VTable::_fresh_host_methods('MyHash');
-is_deeply( $methods->to_perl, ['oodle'], "fresh_host_methods" );
-
-my $folder = RAMFolderOfDeath->new;
-eval { $folder->slurp_file('foo') }; # calls open_in, which dies per above.
-like( $@, qr/sweet/i, "override vtable method with pure perl method" );
[4/4] git commit: refs/heads/264_dont_subclass_hash - Cease
unique-ifying values in SortFieldWriter.
Posted by ma...@apache.org.
Cease unique-ifying values in SortFieldWriter.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/4195da77
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/4195da77
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/4195da77
Branch: refs/heads/264_dont_subclass_hash
Commit: 4195da771d0033754ff0988805a9280b47461916
Parents: 258bb69
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Thu Jul 3 13:13:10 2014 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Thu Jul 3 13:13:10 2014 -0700
----------------------------------------------------------------------
core/Lucy/Index/SortFieldWriter.c | 53 ++++++++------------------------
core/Lucy/Index/SortFieldWriter.cfh | 4 ---
2 files changed, 12 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/4195da77/core/Lucy/Index/SortFieldWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortFieldWriter.c b/core/Lucy/Index/SortFieldWriter.c
index a59ed3d..31b04e8 100644
--- a/core/Lucy/Index/SortFieldWriter.c
+++ b/core/Lucy/Index/SortFieldWriter.c
@@ -133,32 +133,13 @@ SortFieldWriter_init(SortFieldWriter *self, Schema *schema,
ivars->mem_per_entry += VTable_Get_Obj_Alloc_Size(FLOAT64);
ivars->var_width = false;
}
- ivars->uniq_vals = Hash_new(0);
return self;
}
void
-SortFieldWriter_Clear_Buffer_IMP(SortFieldWriter *self) {
- SortFieldWriterIVARS *const ivars = SortFieldWriter_IVARS(self);
- if (ivars->uniq_vals) {
- Hash_Clear(ivars->uniq_vals);
- }
- SortFieldWriter_Clear_Buffer_t super_clear_buffer
- = SUPER_METHOD_PTR(SORTFIELDWRITER, LUCY_SortFieldWriter_Clear_Buffer);
- super_clear_buffer(self);
- // Note that we have not Reset() the Counter which tracks memory usage.
- // This is because the counter is shared amongst multiple SortFieldWriters
- // which belong to a parent SortWriter; it is the responsibility of the
- // parent SortWriter to reset it once **all** of its child
- // SortFieldWriters have cleared their buffers.
-}
-
-void
SortFieldWriter_Destroy_IMP(SortFieldWriter *self) {
SortFieldWriterIVARS *const ivars = SortFieldWriter_IVARS(self);
- DECREF(ivars->uniq_vals);
- ivars->uniq_vals = NULL;
DECREF(ivars->field);
DECREF(ivars->schema);
DECREF(ivars->snapshot);
@@ -191,29 +172,19 @@ SortFieldWriter_Get_Ord_Width_IMP(SortFieldWriter *self) {
void
SortFieldWriter_Add_IMP(SortFieldWriter *self, int32_t doc_id, Obj *value) {
SortFieldWriterIVARS *const ivars = SortFieldWriter_IVARS(self);
- Hash *uniq_vals = ivars->uniq_vals;
Counter *counter = ivars->counter;
-
- // Uniq-ify the value.
- int32_t hash_sum = Obj_Hash_Sum(value);
- Obj *uniq_val = Hash_Find_Key(uniq_vals, value, hash_sum);
- if (!uniq_val) {
- Hash_Store(uniq_vals, value, (Obj*)CFISH_TRUE);
- Counter_Add(counter, ivars->mem_per_entry);
- if (ivars->prim_id == FType_TEXT) {
- int64_t size = Str_Get_Size((String*)value) + 1;
- size = SI_increase_to_word_multiple(size);
- Counter_Add(counter, size);
- }
- else if (ivars->prim_id == FType_BLOB) {
- int64_t size = BB_Get_Size((ByteBuf*)value) + 1;
- size = SI_increase_to_word_multiple(size);
- Counter_Add(counter, size);
- }
- uniq_val = Hash_Find_Key(uniq_vals, value, hash_sum);
+ Counter_Add(counter, ivars->mem_per_entry);
+ if (ivars->prim_id == FType_TEXT) {
+ int64_t size = Str_Get_Size((String*)value) + 1;
+ size = SI_increase_to_word_multiple(size);
+ Counter_Add(counter, size);
}
-
- SFWriterElem *elem = S_SFWriterElem_create(uniq_val, doc_id);
+ else if (ivars->prim_id == FType_BLOB) {
+ int64_t size = BB_Get_Size((ByteBuf*)value) + 1;
+ size = SI_increase_to_word_multiple(size);
+ Counter_Add(counter, size);
+ }
+ SFWriterElem *elem = S_SFWriterElem_create(Obj_Clone(value), doc_id);
SortFieldWriter_Feed(self, (Obj*)elem);
ivars->count++;
}
@@ -770,7 +741,7 @@ static SFWriterElem*
S_SFWriterElem_create(Obj *value, int32_t doc_id) {
SFWriterElem *self = (SFWriterElem*)VTable_Make_Obj(SFWRITERELEM);
SFWriterElemIVARS *ivars = SFWriterElem_IVARS(self);
- ivars->value = INCREF(value);
+ ivars->value = value;
ivars->doc_id = doc_id;
return self;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/4195da77/core/Lucy/Index/SortFieldWriter.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortFieldWriter.cfh b/core/Lucy/Index/SortFieldWriter.cfh
index 22b2d74..a2d0ddc 100644
--- a/core/Lucy/Index/SortFieldWriter.cfh
+++ b/core/Lucy/Index/SortFieldWriter.cfh
@@ -19,7 +19,6 @@ parcel Lucy;
class Lucy::Index::SortFieldWriter
inherits Lucy::Util::SortExternal {
String *field;
- Hash *uniq_vals;
Schema *schema;
Snapshot *snapshot;
Segment *segment;
@@ -86,9 +85,6 @@ class Lucy::Index::SortFieldWriter
int
Compare(SortFieldWriter *self, void *va, void *vb);
- void
- Clear_Buffer(SortFieldWriter *self);
-
int32_t
Get_Null_Ord(SortFieldWriter *self);
[2/4] git commit: refs/heads/264_dont_subclass_hash - Remove unused
var.
Posted by ma...@apache.org.
Remove unused var.
This commit will be consolidated into another before merging to master.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/a833c339
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/a833c339
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/a833c339
Branch: refs/heads/264_dont_subclass_hash
Commit: a833c339a3110f194be0d9717a22d3491c2275ac
Parents: 09336bb
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Thu Jul 3 10:48:27 2014 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Thu Jul 3 10:48:27 2014 -0700
----------------------------------------------------------------------
core/Lucy/Index/SortFieldWriter.cfh | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/a833c339/core/Lucy/Index/SortFieldWriter.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortFieldWriter.cfh b/core/Lucy/Index/SortFieldWriter.cfh
index e2b996f..3625af3 100644
--- a/core/Lucy/Index/SortFieldWriter.cfh
+++ b/core/Lucy/Index/SortFieldWriter.cfh
@@ -50,7 +50,6 @@ class Lucy::Index::SortFieldWriter
int32_t *sorted_ids;
int32_t run_tick;
int32_t ord_width;
- Obj *last_val;
inert incremented SortFieldWriter*
new(Schema *schema, Snapshot *snapshot, Segment *segment,
[3/4] git commit: refs/heads/264_dont_subclass_hash - Improve mem
tracking.
Posted by ma...@apache.org.
Improve mem tracking.
Cache size of value object. Account for size of SFWriterElem. Handle
blob fields better.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/258bb694
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/258bb694
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/258bb694
Branch: refs/heads/264_dont_subclass_hash
Commit: 258bb6949680961aa5906c3b2baac755c2473443
Parents: a833c33
Author: Marvin Humphrey <ma...@rectangular.com>
Authored: Thu Jul 3 12:54:33 2014 -0700
Committer: Marvin Humphrey <ma...@rectangular.com>
Committed: Thu Jul 3 12:54:33 2014 -0700
----------------------------------------------------------------------
core/Lucy/Index/SortFieldWriter.c | 48 +++++++++++++++++++-------------
core/Lucy/Index/SortFieldWriter.cfh | 1 +
2 files changed, 29 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/258bb694/core/Lucy/Index/SortFieldWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortFieldWriter.c b/core/Lucy/Index/SortFieldWriter.c
index fc350ba..a59ed3d 100644
--- a/core/Lucy/Index/SortFieldWriter.c
+++ b/core/Lucy/Index/SortFieldWriter.c
@@ -120,10 +120,17 @@ SortFieldWriter_init(SortFieldWriter *self, Schema *schema,
Schema_Fetch_Type(ivars->schema, field), FIELDTYPE);
ivars->type = (FieldType*)INCREF(type);
ivars->prim_id = FType_Primitive_ID(type);
- if (ivars->prim_id == FType_TEXT || ivars->prim_id == FType_BLOB) {
+ ivars->mem_per_entry = VTable_Get_Obj_Alloc_Size(SFWRITERELEM);
+ if (ivars->prim_id == FType_TEXT) {
+ ivars->mem_per_entry += VTable_Get_Obj_Alloc_Size(STRING);
+ ivars->var_width = true;
+ }
+ else if (ivars->prim_id == FType_BLOB) {
+ ivars->mem_per_entry += VTable_Get_Obj_Alloc_Size(BYTEBUF);
ivars->var_width = true;
}
else {
+ ivars->mem_per_entry += VTable_Get_Obj_Alloc_Size(FLOAT64);
ivars->var_width = false;
}
ivars->uniq_vals = Hash_new(0);
@@ -181,31 +188,32 @@ SortFieldWriter_Get_Ord_Width_IMP(SortFieldWriter *self) {
return SortFieldWriter_IVARS(self)->ord_width;
}
-static Obj*
-S_find_unique_value(Hash *uniq_vals, Counter *counter, Obj *val) {
- int32_t hash_sum = Obj_Hash_Sum(val);
- Obj *uniq_val = Hash_Find_Key(uniq_vals, val, hash_sum);
+void
+SortFieldWriter_Add_IMP(SortFieldWriter *self, int32_t doc_id, Obj *value) {
+ SortFieldWriterIVARS *const ivars = SortFieldWriter_IVARS(self);
+ Hash *uniq_vals = ivars->uniq_vals;
+ Counter *counter = ivars->counter;
+
+ // Uniq-ify the value.
+ int32_t hash_sum = Obj_Hash_Sum(value);
+ Obj *uniq_val = Hash_Find_Key(uniq_vals, value, hash_sum);
if (!uniq_val) {
- Hash_Store(uniq_vals, val, (Obj*)CFISH_TRUE);
- VTable *vtable = Obj_Get_VTable(val);
- Counter_Add(counter, VTable_Get_Obj_Alloc_Size(vtable));
- if (vtable == STRING) {
- int64_t size = Str_Get_Size((String*)val) + 1;
+ Hash_Store(uniq_vals, value, (Obj*)CFISH_TRUE);
+ Counter_Add(counter, ivars->mem_per_entry);
+ if (ivars->prim_id == FType_TEXT) {
+ int64_t size = Str_Get_Size((String*)value) + 1;
+ size = SI_increase_to_word_multiple(size);
+ Counter_Add(counter, size);
+ }
+ else if (ivars->prim_id == FType_BLOB) {
+ int64_t size = BB_Get_Size((ByteBuf*)value) + 1;
size = SI_increase_to_word_multiple(size);
Counter_Add(counter, size);
}
- uniq_val = Hash_Find_Key(uniq_vals, val, hash_sum);
+ uniq_val = Hash_Find_Key(uniq_vals, value, hash_sum);
}
- return uniq_val;
-}
-
-void
-SortFieldWriter_Add_IMP(SortFieldWriter *self, int32_t doc_id, Obj *value) {
- SortFieldWriterIVARS *const ivars = SortFieldWriter_IVARS(self);
- // Uniq-ify the value, and record it for this document.
- Obj *copy = S_find_unique_value(ivars->uniq_vals, ivars->counter, value);
- SFWriterElem *elem = S_SFWriterElem_create(copy, doc_id);
+ SFWriterElem *elem = S_SFWriterElem_create(uniq_val, doc_id);
SortFieldWriter_Feed(self, (Obj*)elem);
ivars->count++;
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/258bb694/core/Lucy/Index/SortFieldWriter.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortFieldWriter.cfh b/core/Lucy/Index/SortFieldWriter.cfh
index 3625af3..22b2d74 100644
--- a/core/Lucy/Index/SortFieldWriter.cfh
+++ b/core/Lucy/Index/SortFieldWriter.cfh
@@ -29,6 +29,7 @@ class Lucy::Index::SortFieldWriter
Counter *counter;
int32_t field_num;
int32_t null_ord;
+ size_t mem_per_entry;
int8_t prim_id;
int32_t count;
OutStream *temp_ord_out;