You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by nw...@apache.org on 2015/04/16 22:42:51 UTC
lucy git commit: Switch over to HashIterator
Repository: lucy
Updated Branches:
refs/heads/CLOWNFISH-2-hash-iterator [created] 42c878f36
Switch over to HashIterator
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/42c878f3
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/42c878f3
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/42c878f3
Branch: refs/heads/CLOWNFISH-2-hash-iterator
Commit: 42c878f36c04b45e15603dfc004116945eaf428f
Parents: 8e232df
Author: Nick Wellnhofer <we...@aevum.de>
Authored: Thu Apr 16 22:41:59 2015 +0200
Committer: Nick Wellnhofer <we...@aevum.de>
Committed: Thu Apr 16 22:41:59 2015 +0200
----------------------------------------------------------------------
c/src/Lucy/Index/Inverter.c | 12 ++++----
core/Lucy/Index/BackgroundMerger.c | 11 ++++---
core/Lucy/Index/DeletionsWriter.c | 10 ++++---
core/Lucy/Index/IndexReader.c | 9 +++---
core/Lucy/Index/PolyReader.c | 19 +++++++-----
core/Lucy/Plan/Schema.c | 34 ++++++++++++----------
core/Lucy/Store/CompoundFileWriter.c | 8 ++---
core/Lucy/Test/Analysis/TestSnowballStemmer.c | 10 ++++---
core/Lucy/Test/Store/TestCompoundFileWriter.c | 15 +++++-----
core/Lucy/Util/Freezer.c | 33 ++++++++++++---------
10 files changed, 91 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucy/blob/42c878f3/c/src/Lucy/Index/Inverter.c
----------------------------------------------------------------------
diff --git a/c/src/Lucy/Index/Inverter.c b/c/src/Lucy/Index/Inverter.c
index 29521d2..1719c4c 100644
--- a/c/src/Lucy/Index/Inverter.c
+++ b/c/src/Lucy/Index/Inverter.c
@@ -24,6 +24,7 @@
#include "Clownfish/String.h"
#include "Clownfish/Err.h"
#include "Clownfish/Hash.h"
+#include "Clownfish/HashIterator.h"
#include "Clownfish/Num.h"
#include "Clownfish/VArray.h"
#include "Lucy/Document/Doc.h"
@@ -62,16 +63,16 @@ void
Inverter_Invert_Doc_IMP(Inverter *self, Doc *doc) {
InverterIVARS *const ivars = Inverter_IVARS(self);
Hash *const fields = (Hash*)Doc_Get_Fields(doc);
- uint32_t num_keys = Hash_Iterate(fields);
// Prepare for the new doc.
Inverter_Set_Doc(self, doc);
// Extract and invert the doc's fields.
- while (num_keys--) {
- String *field;
- Obj *obj;
- Hash_Next(fields, &field, &obj);
+ HashIterator *iter = HashIter_new(fields);
+ while (HashIter_Next(iter)) {
+ String *field = HashIter_Get_Key(iter);
+ Obj *obj = HashIter_Get_Value(iter);
+
InverterEntry *inventry = S_fetch_entry(ivars, field);
InverterEntryIVARS *inventry_ivars = InvEntry_IVARS(inventry);
FieldType *type = inventry_ivars->type;
@@ -122,6 +123,7 @@ Inverter_Invert_Doc_IMP(Inverter *self, Doc *doc) {
Inverter_Add_Field(self, inventry);
}
+ DECREF(iter);
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/42c878f3/core/Lucy/Index/BackgroundMerger.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/BackgroundMerger.c b/core/Lucy/Index/BackgroundMerger.c
index d39480d..93b4996 100644
--- a/core/Lucy/Index/BackgroundMerger.c
+++ b/core/Lucy/Index/BackgroundMerger.c
@@ -17,6 +17,7 @@
#define C_LUCY_BACKGROUNDMERGER
#include "Lucy/Util/ToolSet.h"
+#include "Clownfish/HashIterator.h"
#include "Lucy/Index/BackgroundMerger.h"
#include "Lucy/Index/DeletionsReader.h"
#include "Lucy/Index/DeletionsWriter.h"
@@ -315,8 +316,6 @@ S_merge_updated_deletions(BackgroundMerger *self) {
int64_t merge_seg_num = Seg_Get_Number(ivars->segment);
uint32_t seg_tick = INT32_MAX;
int32_t offset = INT32_MAX;
- String *seg_name = NULL;
- Matcher *deletions = NULL;
SegWriter_Prep_Seg_Dir(seg_writer);
@@ -332,8 +331,11 @@ S_merge_updated_deletions(BackgroundMerger *self) {
}
if (offset == INT32_MAX) { THROW(ERR, "Failed sanity check"); }
- Hash_Iterate(updated_deletions);
- while (Hash_Next(updated_deletions, &seg_name, (Obj**)&deletions)) {
+ HashIterator *iter = HashIter_new(updated_deletions);
+ while (HashIter_Next(iter)) {
+ String *seg_name = HashIter_Get_Key(iter);
+ Matcher *deletions = (Matcher*)HashIter_Get_Value(iter);
+
I32Array *doc_map
= (I32Array*)CERTIFY(
Hash_Fetch(ivars->doc_maps, seg_name),
@@ -351,6 +353,7 @@ S_merge_updated_deletions(BackgroundMerger *self) {
}
}
}
+ DECREF(iter);
// Finish the segment and clean up.
DelWriter_Finish(del_writer);
http://git-wip-us.apache.org/repos/asf/lucy/blob/42c878f3/core/Lucy/Index/DeletionsWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/DeletionsWriter.c b/core/Lucy/Index/DeletionsWriter.c
index 77e4974..9e088c7 100644
--- a/core/Lucy/Index/DeletionsWriter.c
+++ b/core/Lucy/Index/DeletionsWriter.c
@@ -20,6 +20,7 @@
#include <math.h>
+#include "Clownfish/HashIterator.h"
#include "Lucy/Index/DeletionsWriter.h"
#include "Lucy/Index/DeletionsReader.h"
#include "Lucy/Index/IndexReader.h"
@@ -350,10 +351,10 @@ DefDelWriter_Merge_Segment_IMP(DefaultDeletionsWriter *self,
VArray *seg_readers = ivars->seg_readers;
Hash *files = (Hash*)Hash_Fetch_Utf8(del_meta, "files", 5);
if (files) {
- String *seg;
- Hash *mini_meta;
- Hash_Iterate(files);
- while (Hash_Next(files, &seg, (Obj**)&mini_meta)) {
+ HashIterator *iter = HashIter_new(files);
+ while (HashIter_Next(iter)) {
+ String *seg = HashIter_Get_Key(iter);
+ Hash *mini_meta = (Hash*)HashIter_Get_Value(iter);
/* Find the segment the deletions from the SegReader
* we're adding correspond to. If it's gone, we don't
@@ -381,6 +382,7 @@ DefDelWriter_Merge_Segment_IMP(DefaultDeletionsWriter *self,
}
}
}
+ DECREF(iter);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/42c878f3/core/Lucy/Index/IndexReader.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/IndexReader.c b/core/Lucy/Index/IndexReader.c
index 459bef9..b236fee 100644
--- a/core/Lucy/Index/IndexReader.c
+++ b/core/Lucy/Index/IndexReader.c
@@ -17,6 +17,7 @@
#define C_LUCY_INDEXREADER
#include "Lucy/Util/ToolSet.h"
+#include "Clownfish/HashIterator.h"
#include "Lucy/Index/IndexReader.h"
#include "Lucy/Index/DocVector.h"
#include "Lucy/Index/IndexManager.h"
@@ -70,14 +71,14 @@ void
IxReader_Close_IMP(IndexReader *self) {
IndexReaderIVARS *const ivars = IxReader_IVARS(self);
if (ivars->components) {
- String *key;
- DataReader *component;
- Hash_Iterate(ivars->components);
- while (Hash_Next(ivars->components, &key, (Obj**)&component)) {
+ HashIterator *iter = HashIter_new(ivars->components);
+ while (HashIter_Next(iter)) {
+ DataReader *component = (DataReader*)HashIter_Get_Value(iter);
if (Obj_Is_A((Obj*)component, DATAREADER)) {
DataReader_Close(component);
}
}
+ DECREF(iter);
Hash_Clear(ivars->components);
}
if (ivars->read_lock) {
http://git-wip-us.apache.org/repos/asf/lucy/blob/42c878f3/core/Lucy/Index/PolyReader.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/PolyReader.c b/core/Lucy/Index/PolyReader.c
index e941a61..203a752 100644
--- a/core/Lucy/Index/PolyReader.c
+++ b/core/Lucy/Index/PolyReader.c
@@ -17,6 +17,7 @@
#define C_LUCY_POLYREADER
#include "Lucy/Util/ToolSet.h"
+#include "Clownfish/HashIterator.h"
#include "Lucy/Index/PolyReader.h"
#include "Lucy/Document/HitDoc.h"
#include "Lucy/Index/DeletionsReader.h"
@@ -115,12 +116,12 @@ S_init_sub_readers(PolyReader *self, VArray *sub_readers) {
for (uint32_t i = 0; i < num_sub_readers; i++) {
SegReader *seg_reader = (SegReader*)VA_Fetch(sub_readers, i);
Hash *components = SegReader_Get_Components(seg_reader);
- String *api;
- DataReader *component;
starts[i] = ivars->doc_max;
ivars->doc_max += SegReader_Doc_Max(seg_reader);
- Hash_Iterate(components);
- while (Hash_Next(components, &api, (Obj**)&component)) {
+ HashIterator *iter = HashIter_new(components);
+ while (HashIter_Next(iter)) {
+ String *api = HashIter_Get_Key(iter);
+ DataReader *component = (DataReader*)HashIter_Get_Value(iter);
VArray *readers = (VArray*)Hash_Fetch(data_readers, api);
if (!readers) {
readers = VA_new(num_sub_readers);
@@ -128,13 +129,14 @@ S_init_sub_readers(PolyReader *self, VArray *sub_readers) {
}
VA_Store(readers, i, INCREF(component));
}
+ DECREF(iter);
}
ivars->offsets = I32Arr_new_steal(starts, num_sub_readers);
- String *api;
- VArray *readers;
- Hash_Iterate(data_readers);
- while (Hash_Next(data_readers, &api, (Obj**)&readers)) {
+ HashIterator *iter = HashIter_new(data_readers);
+ while (HashIter_Next(iter)) {
+ String *api = HashIter_Get_Key(iter);
+ VArray *readers = (VArray*)HashIter_Get_Value(iter);
DataReader *datareader
= (DataReader*)CERTIFY(S_first_non_null(readers), DATAREADER);
DataReader *aggregator
@@ -144,6 +146,7 @@ S_init_sub_readers(PolyReader *self, VArray *sub_readers) {
Hash_Store(ivars->components, api, (Obj*)aggregator);
}
}
+ DECREF(iter);
DECREF(data_readers);
DeletionsReader *del_reader
http://git-wip-us.apache.org/repos/asf/lucy/blob/42c878f3/core/Lucy/Plan/Schema.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Plan/Schema.c b/core/Lucy/Plan/Schema.c
index 0bf9842..83feaa0 100644
--- a/core/Lucy/Plan/Schema.c
+++ b/core/Lucy/Plan/Schema.c
@@ -19,6 +19,7 @@
#include <ctype.h>
#include "Lucy/Util/ToolSet.h"
+#include "Clownfish/HashIterator.h"
#include "Lucy/Plan/Schema.h"
#include "Lucy/Analysis/Analyzer.h"
#include "Lucy/Index/Similarity.h"
@@ -252,8 +253,6 @@ Schema_Dump_IMP(Schema *self) {
SchemaIVARS *const ivars = Schema_IVARS(self);
Hash *dump = Hash_new(0);
Hash *type_dumps = Hash_new(Hash_Get_Size(ivars->types));
- String *field;
- FieldType *type;
// Record class name, store dumps of unique Analyzers.
Hash_Store_Utf8(dump, "_class", 6,
@@ -263,9 +262,11 @@ Schema_Dump_IMP(Schema *self) {
// Dump FieldTypes.
Hash_Store_Utf8(dump, "fields", 6, (Obj*)type_dumps);
- Hash_Iterate(ivars->types);
- while (Hash_Next(ivars->types, &field, (Obj**)&type)) {
- Class *type_class = FType_Get_Class(type);
+ HashIterator *iter = HashIter_new(ivars->types);
+ while (HashIter_Next(iter)) {
+ String *field = HashIter_Get_Key(iter);
+ FieldType *type = (FieldType*)HashIter_Get_Value(iter);
+ Class *type_class = FType_Get_Class(type);
// Dump known types to simplified format.
if (type_class == FULLTEXTTYPE) {
@@ -290,6 +291,7 @@ Schema_Dump_IMP(Schema *self) {
Hash_Store(type_dumps, field, FType_Dump(type));
}
}
+ DECREF(iter);
return dump;
}
@@ -315,8 +317,6 @@ Schema_Load_IMP(Schema *self, Obj *dump) {
= (VArray*)CERTIFY(Hash_Fetch_Utf8(source, "analyzers", 9), VARRAY);
VArray *analyzers
= (VArray*)Freezer_load((Obj*)analyzer_dumps);
- String *field;
- Hash *type_dump;
UNUSED_VAR(self);
// Start with a blank Schema.
@@ -324,11 +324,11 @@ Schema_Load_IMP(Schema *self, Obj *dump) {
SchemaIVARS *const loaded_ivars = Schema_IVARS(loaded);
VA_Grow(loaded_ivars->uniq_analyzers, VA_Get_Size(analyzers));
- Hash_Iterate(type_dumps);
- while (Hash_Next(type_dumps, &field, (Obj**)&type_dump)) {
- String *type_str;
- CERTIFY(type_dump, HASH);
- type_str = (String*)Hash_Fetch_Utf8(type_dump, "type", 4);
+ HashIterator *iter = HashIter_new(type_dumps);
+ while (HashIter_Next(iter)) {
+ String *field = HashIter_Get_Key(iter);
+ Hash *type_dump = (Hash*)CERTIFY(HashIter_Get_Value(iter), HASH);
+ String *type_str = (String*)Hash_Fetch_Utf8(type_dump, "type", 4);
if (type_str) {
if (Str_Equals_Utf8(type_str, "fulltext", 8)) {
// Replace the "analyzer" tick with the real thing.
@@ -395,6 +395,7 @@ Schema_Load_IMP(Schema *self, Obj *dump) {
DECREF(type);
}
}
+ DECREF(iter);
DECREF(analyzers);
@@ -408,13 +409,14 @@ Schema_Eat_IMP(Schema *self, Schema *other) {
Schema_Get_Class_Name(self), Schema_Get_Class_Name(other));
}
- String *field;
- FieldType *type;
SchemaIVARS *const ovars = Schema_IVARS(other);
- Hash_Iterate(ovars->types);
- while (Hash_Next(ovars->types, &field, (Obj**)&type)) {
+ HashIterator *iter = HashIter_new(ovars->types);
+ while (HashIter_Next(iter)) {
+ String *field = HashIter_Get_Key(iter);
+ FieldType *type = (FieldType*)HashIter_Get_Value(iter);
Schema_Spec_Field(self, field, type);
}
+ DECREF(iter);
}
void
http://git-wip-us.apache.org/repos/asf/lucy/blob/42c878f3/core/Lucy/Store/CompoundFileWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Store/CompoundFileWriter.c b/core/Lucy/Store/CompoundFileWriter.c
index b3b137e..51749ee 100644
--- a/core/Lucy/Store/CompoundFileWriter.c
+++ b/core/Lucy/Store/CompoundFileWriter.c
@@ -155,16 +155,16 @@ S_do_consolidate(CompoundFileWriter *self, CompoundFileWriterIVARS *ivars) {
DECREF(files);
DECREF(metadata);
/*
- String *merged_file;
- Obj *ignore;
- Hash_Iterate(sub_files);
- while (Hash_Next(sub_files, &merged_file, &ignore)) {
+ HashIterator *iter = HashIter_new(sub_files);
+ while (HashIter_Next(iter)) {
+ String *merged_file = HashIter_Get_Key(iter);
if (!Folder_Delete(folder, merged_file)) {
String *mess = MAKE_MESS("Can't delete '%o'", merged_file);
DECREF(sub_files);
Err_throw_mess(ERR, mess);
}
}
+ DECREF(iter);
*/
DECREF(sub_files);
for (uint32_t i = 0, max = VA_Get_Size(merged); i < max; i++) {
http://git-wip-us.apache.org/repos/asf/lucy/blob/42c878f3/core/Lucy/Test/Analysis/TestSnowballStemmer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Test/Analysis/TestSnowballStemmer.c b/core/Lucy/Test/Analysis/TestSnowballStemmer.c
index d5cf7ce..18e6a78 100644
--- a/core/Lucy/Test/Analysis/TestSnowballStemmer.c
+++ b/core/Lucy/Test/Analysis/TestSnowballStemmer.c
@@ -18,6 +18,7 @@
#define TESTLUCY_USE_SHORT_NAMES
#include "Lucy/Util/ToolSet.h"
+#include "Clownfish/HashIterator.h"
#include "Clownfish/TestHarness/TestBatchRunner.h"
#include "Lucy/Test.h"
#include "Lucy/Test/Analysis/TestSnowballStemmer.h"
@@ -67,10 +68,10 @@ test_stemming(TestBatchRunner *runner) {
Hash *tests = (Hash*)Json_slurp_json((Folder*)modules_folder, path);
if (!tests) { RETHROW(Err_get_error()); }
- String *iso;
- Hash *lang_data;
- Hash_Iterate(tests);
- while (Hash_Next(tests, &iso, (Obj**)&lang_data)) {
+ HashIterator *iter = HashIter_new(tests);
+ while (HashIter_Next(iter)) {
+ String *iso = HashIter_Get_Key(iter);
+ Hash *lang_data = (Hash*)HashIter_Get_Value(iter);
VArray *words = (VArray*)Hash_Fetch_Utf8(lang_data, "words", 5);
VArray *stems = (VArray*)Hash_Fetch_Utf8(lang_data, "stems", 5);
SnowballStemmer *stemmer = SnowStemmer_new(iso);
@@ -88,6 +89,7 @@ test_stemming(TestBatchRunner *runner) {
}
DECREF(stemmer);
}
+ DECREF(iter);
DECREF(tests);
DECREF(modules_folder);
http://git-wip-us.apache.org/repos/asf/lucy/blob/42c878f3/core/Lucy/Test/Store/TestCompoundFileWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Test/Store/TestCompoundFileWriter.c b/core/Lucy/Test/Store/TestCompoundFileWriter.c
index 2cce467..c633a83 100644
--- a/core/Lucy/Test/Store/TestCompoundFileWriter.c
+++ b/core/Lucy/Test/Store/TestCompoundFileWriter.c
@@ -19,6 +19,7 @@
#include "charmony.h"
+#include "Clownfish/HashIterator.h"
#include "Clownfish/TestHarness/TestBatchRunner.h"
#include "Lucy/Test.h"
#include "Lucy/Test/Store/TestCompoundFileWriter.h"
@@ -118,17 +119,16 @@ test_offsets(TestBatchRunner *runner) {
files = (Hash*)CERTIFY(
Hash_Fetch_Utf8(cf_metadata, "files", 5), HASH);
- String *file;
- Obj *filestats;
bool offsets_ok = true;
TEST_TRUE(runner, Hash_Get_Size(files) > 0, "Multiple files");
- Hash_Iterate(files);
- while (Hash_Next(files, &file, &filestats)) {
- Hash *stats = (Hash*)CERTIFY(filestats, HASH);
- Obj *offset = CERTIFY(Hash_Fetch_Utf8(stats, "offset", 6), OBJ);
- int64_t offs = Obj_To_I64(offset);
+ HashIterator *iter = HashIter_new(files);
+ while (HashIter_Next(iter)) {
+ String *file = HashIter_Get_Key(iter);
+ Hash *stats = (Hash*)CERTIFY(HashIter_Get_Value(iter), HASH);
+ Obj *offset = CERTIFY(Hash_Fetch_Utf8(stats, "offset", 6), OBJ);
+ int64_t offs = Obj_To_I64(offset);
if (offs % 8 != 0) {
offsets_ok = false;
FAIL(runner, "Offset %" PRId64 " for %s not a multiple of 8",
@@ -136,6 +136,7 @@ test_offsets(TestBatchRunner *runner) {
break;
}
}
+ DECREF(iter);
if (offsets_ok) {
PASS(runner, "All offsets are multiples of 8");
}
http://git-wip-us.apache.org/repos/asf/lucy/blob/42c878f3/core/Lucy/Util/Freezer.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Util/Freezer.c b/core/Lucy/Util/Freezer.c
index dce1884..784d62d 100644
--- a/core/Lucy/Util/Freezer.c
+++ b/core/Lucy/Util/Freezer.c
@@ -17,6 +17,7 @@
#define C_LUCY_FREEZER
#include "Lucy/Util/ToolSet.h"
+#include "Clownfish/HashIterator.h"
#include "Lucy/Util/Freezer.h"
#include "Lucy/Store/InStream.h"
#include "Lucy/Store/OutStream.h"
@@ -288,16 +289,17 @@ Freezer_read_varray(InStream *instream) {
void
Freezer_serialize_hash(Hash *hash, OutStream *outstream) {
- String *key;
- Obj *val;
uint32_t hash_size = Hash_Get_Size(hash);
OutStream_Write_C32(outstream, hash_size);
- Hash_Iterate(hash);
- while (Hash_Next(hash, &key, &val)) {
+ HashIterator *iter = HashIter_new(hash);
+ while (HashIter_Next(iter)) {
+ String *key = HashIter_Get_Key(iter);
+ Obj *val = HashIter_Get_Value(iter);
Freezer_serialize_string(key, outstream);
FREEZE(val, outstream);
}
+ DECREF(iter);
}
Hash*
@@ -339,14 +341,15 @@ S_dump_array(VArray *array) {
Obj*
S_dump_hash(Hash *hash) {
- Hash *dump = Hash_new(Hash_Get_Size(hash));
- String *key;
- Obj *value;
+ Hash *dump = Hash_new(Hash_Get_Size(hash));
- Hash_Iterate(hash);
- while (Hash_Next(hash, &key, &value)) {
+ HashIterator *iter = HashIter_new(hash);
+ while (HashIter_Next(iter)) {
+ String *key = HashIter_Get_Key(iter);
+ Obj *value = HashIter_Get_Value(iter);
Hash_Store(dump, key, Freezer_dump(value));
}
+ DECREF(iter);
return (Obj*)dump;
}
@@ -447,13 +450,15 @@ S_load_from_hash(Hash *dump) {
}
// It's an ordinary Hash.
- Hash *loaded = Hash_new(Hash_Get_Size(dump));
- String *key;
- Obj *value;
- Hash_Iterate(dump);
- while (Hash_Next(dump, &key, &value)) {
+ Hash *loaded = Hash_new(Hash_Get_Size(dump));
+
+ HashIterator *iter = HashIter_new(dump);
+ while (HashIter_Next(iter)) {
+ String *key = HashIter_Get_Key(iter);
+ Obj *value = HashIter_Get_Value(iter);
Hash_Store(loaded, key, Freezer_load(value));
}
+ DECREF(iter);
return (Obj*)loaded;