You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by mi...@apache.org on 2010/10/14 10:48:40 UTC
[lucy-commits] svn commit: r1022422 [10/43] - in /incubator/lucy/trunk/import: ./
charmonizer/ charmonizer/src/ charmonizer/src/Charmonizer/
charmonizer/src/Charmonizer/Core/ charmonizer/src/Charmonizer/Probe/
charmonizer/src/Charmonizer/Test/ clownfish/ clownfish/li...
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/DeletionsWriter.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/DeletionsWriter.c?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/DeletionsWriter.c (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/DeletionsWriter.c Thu Oct 14 08:48:04 2010
@@ -0,0 +1,383 @@
+#define C_KINO_DELETIONSWRITER
+#define C_KINO_DEFAULTDELETIONSWRITER
+#include "KinoSearch/Util/ToolSet.h"
+
+#include <math.h>
+
+#include "KinoSearch/Index/DeletionsWriter.h"
+#include "KinoSearch/Index/DeletionsReader.h"
+#include "KinoSearch/Index/IndexReader.h"
+#include "KinoSearch/Index/PolyReader.h"
+#include "KinoSearch/Index/PostingList.h"
+#include "KinoSearch/Index/PostingListReader.h"
+#include "KinoSearch/Index/Segment.h"
+#include "KinoSearch/Index/SegReader.h"
+#include "KinoSearch/Index/Snapshot.h"
+#include "KinoSearch/Plan/Schema.h"
+#include "KinoSearch/Search/BitVecMatcher.h"
+#include "KinoSearch/Search/Compiler.h"
+#include "KinoSearch/Search/IndexSearcher.h"
+#include "KinoSearch/Search/Matcher.h"
+#include "KinoSearch/Search/Query.h"
+#include "KinoSearch/Store/Folder.h"
+#include "KinoSearch/Store/OutStream.h"
+
+DeletionsWriter*
+DelWriter_init(DeletionsWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader)
+{
+ DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
+ ABSTRACT_CLASS_CHECK(self, DELETIONSWRITER);
+ return self;
+}
+
+I32Array*
+DelWriter_generate_doc_map(DeletionsWriter *self, Matcher *deletions,
+ int32_t doc_max, int32_t offset)
+{
+ int32_t *doc_map = (int32_t*)CALLOCATE(doc_max + 1, sizeof(int32_t));
+ int32_t new_doc_id;
+ int32_t i;
+ int32_t next_deletion = deletions ? Matcher_Next(deletions) : I32_MAX;
+ UNUSED_VAR(self);
+
+ // 0 for a deleted doc, a new number otherwise
+ for (i = 1, new_doc_id = 1; i <= doc_max; i++) {
+ if (i == next_deletion) {
+ next_deletion = Matcher_Next(deletions);
+ }
+ else {
+ doc_map[i] = offset + new_doc_id++;
+ }
+ }
+
+ return I32Arr_new_steal(doc_map, doc_max + 1);
+}
+
+int32_t DefDelWriter_current_file_format = 1;
+
+DefaultDeletionsWriter*
+DefDelWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader)
+{
+ DefaultDeletionsWriter *self
+ = (DefaultDeletionsWriter*)VTable_Make_Obj(DEFAULTDELETIONSWRITER);
+ return DefDelWriter_init(self, schema, snapshot, segment, polyreader);
+}
+
+DefaultDeletionsWriter*
+DefDelWriter_init(DefaultDeletionsWriter *self, Schema *schema,
+ Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader)
+{
+ uint32_t i;
+ uint32_t num_seg_readers;
+
+ DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
+ self->seg_readers = PolyReader_Seg_Readers(polyreader);
+ num_seg_readers = VA_Get_Size(self->seg_readers);
+ self->seg_starts = PolyReader_Offsets(polyreader);
+ self->bit_vecs = VA_new(num_seg_readers);
+ self->updated = (bool_t*)CALLOCATE(num_seg_readers, sizeof(bool_t));
+ self->searcher = IxSearcher_new((Obj*)polyreader);
+ self->name_to_tick = Hash_new(num_seg_readers);
+
+ // Materialize a BitVector of deletions for each segment.
+ for (i = 0; i < num_seg_readers; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(self->seg_readers, i);
+ BitVector *bit_vec = BitVec_new(SegReader_Doc_Max(seg_reader));
+ DeletionsReader *del_reader = (DeletionsReader*)SegReader_Fetch(
+ seg_reader, VTable_Get_Name(DELETIONSREADER));
+ Matcher *seg_dels = del_reader
+ ? DelReader_Iterator(del_reader) : NULL;
+
+ if (seg_dels) {
+ int32_t del;
+ while (0 != (del = Matcher_Next(seg_dels))) {
+ BitVec_Set(bit_vec, del);
+ }
+ DECREF(seg_dels);
+ }
+ VA_Store(self->bit_vecs, i, (Obj*)bit_vec);
+ Hash_Store(self->name_to_tick,
+ (Obj*)SegReader_Get_Seg_Name(seg_reader), (Obj*)Int32_new(i));
+ }
+
+ return self;
+}
+
+void
+DefDelWriter_destroy(DefaultDeletionsWriter *self)
+{
+ DECREF(self->seg_readers);
+ DECREF(self->seg_starts);
+ DECREF(self->bit_vecs);
+ DECREF(self->searcher);
+ DECREF(self->name_to_tick);
+ FREEMEM(self->updated);
+ SUPER_DESTROY(self, DEFAULTDELETIONSWRITER);
+}
+
+static CharBuf*
+S_del_filename(DefaultDeletionsWriter *self, SegReader *target_reader)
+{
+ Segment *target_seg = SegReader_Get_Segment(target_reader);
+ return CB_newf("%o/deletions-%o.bv", Seg_Get_Name(self->segment),
+ Seg_Get_Name(target_seg));
+}
+
+void
+DefDelWriter_finish(DefaultDeletionsWriter *self)
+{
+ Folder *const folder = self->folder;
+ uint32_t i, max;
+
+ for (i = 0, max = VA_Get_Size(self->seg_readers); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(self->seg_readers, i);
+ if (self->updated[i]) {
+ BitVector *deldocs = (BitVector*)VA_Fetch(self->bit_vecs, i);
+ int32_t doc_max = SegReader_Doc_Max(seg_reader);
+ double used = (doc_max + 1) / 8.0;
+ uint32_t byte_size = (uint32_t)ceil(used);
+ uint32_t new_max = byte_size * 8 - 1;
+ CharBuf *filename = S_del_filename(self, seg_reader);
+ OutStream *outstream = Folder_Open_Out(folder, filename);
+ if (!outstream) { RETHROW(INCREF(Err_get_error())); }
+
+ // Ensure that we have 1 bit for each doc in segment.
+ BitVec_Grow(deldocs, new_max);
+
+ // Write deletions data and clean up.
+ OutStream_Write_Bytes(outstream,
+ (char*)BitVec_Get_Raw_Bits(deldocs), byte_size);
+ OutStream_Close(outstream);
+ DECREF(outstream);
+ DECREF(filename);
+ }
+ }
+
+ Seg_Store_Metadata_Str(self->segment, "deletions", 9,
+ (Obj*)DefDelWriter_Metadata(self));
+}
+
+Hash*
+DefDelWriter_metadata(DefaultDeletionsWriter *self)
+{
+ Hash *const metadata = DataWriter_metadata((DataWriter*)self);
+ Hash *const files = Hash_new(0);
+ uint32_t i, max;
+
+ for (i = 0, max = VA_Get_Size(self->seg_readers); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(self->seg_readers, i);
+ if (self->updated[i]) {
+ BitVector *deldocs = (BitVector*)VA_Fetch(self->bit_vecs, i);
+ Segment *segment = SegReader_Get_Segment(seg_reader);
+ Hash *mini_meta = Hash_new(2);
+ Hash_Store_Str(mini_meta, "count", 5,
+ (Obj*)CB_newf("%u32", (uint32_t)BitVec_Count(deldocs)) );
+ Hash_Store_Str(mini_meta, "filename", 8,
+ (Obj*)S_del_filename(self, seg_reader));
+ Hash_Store(files, (Obj*)Seg_Get_Name(segment), (Obj*)mini_meta);
+ }
+ }
+ Hash_Store_Str(metadata, "files", 5, (Obj*)files);
+
+ return metadata;
+}
+
+int32_t
+DefDelWriter_format(DefaultDeletionsWriter *self)
+{
+ UNUSED_VAR(self);
+ return DefDelWriter_current_file_format;
+}
+
+Matcher*
+DefDelWriter_seg_deletions(DefaultDeletionsWriter *self,
+ SegReader *seg_reader)
+{
+ Matcher *deletions = NULL;
+ Segment *segment = SegReader_Get_Segment(seg_reader);
+ CharBuf *seg_name = Seg_Get_Name(segment);
+ Integer32 *tick_obj = (Integer32*)Hash_Fetch(self->name_to_tick,
+ (Obj*)seg_name);
+ int32_t tick = tick_obj ? Int32_Get_Value(tick_obj) : 0;
+ SegReader *candidate = tick_obj
+ ? (SegReader*)VA_Fetch(self->seg_readers, tick)
+ : NULL;
+
+ if (tick_obj) {
+ DeletionsReader *del_reader = (DeletionsReader*)SegReader_Obtain(
+ candidate, VTable_Get_Name(DELETIONSREADER));
+ if (self->updated[tick] || DelReader_Del_Count(del_reader)) {
+ BitVector *deldocs = (BitVector*)VA_Fetch(self->bit_vecs, tick);
+ deletions = (Matcher*)BitVecMatcher_new(deldocs);
+ }
+ }
+ else { // Sanity check.
+ THROW(ERR, "Couldn't find SegReader %o", seg_reader);
+ }
+
+ return deletions;
+}
+
+int32_t
+DefDelWriter_seg_del_count(DefaultDeletionsWriter *self,
+ const CharBuf *seg_name)
+{
+ Integer32 *tick
+ = (Integer32*)Hash_Fetch(self->name_to_tick, (Obj*)seg_name);
+ BitVector *deldocs = tick
+ ? (BitVector*)VA_Fetch(self->bit_vecs, Int32_Get_Value(tick))
+ : NULL;
+ return deldocs ? BitVec_Count(deldocs) : 0;
+}
+
+void
+DefDelWriter_delete_by_term(DefaultDeletionsWriter *self,
+ const CharBuf *field, Obj *term)
+{
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->seg_readers); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(self->seg_readers, i);
+ PostingListReader *plist_reader = (PostingListReader*)SegReader_Fetch(
+ seg_reader, VTable_Get_Name(POSTINGLISTREADER));
+ BitVector *bit_vec = (BitVector*)VA_Fetch(self->bit_vecs, i);
+ PostingList *plist = plist_reader
+ ? PListReader_Posting_List(plist_reader, field, term) : NULL;
+ int32_t doc_id;
+ int32_t num_zapped = 0;
+
+ // Iterate through postings, marking each doc as deleted.
+ if (plist) {
+ while (0 != (doc_id = PList_Next(plist))) {
+ num_zapped += !BitVec_Get(bit_vec, doc_id);
+ BitVec_Set(bit_vec, doc_id);
+ }
+ if (num_zapped) { self->updated[i] = true; }
+ DECREF(plist);
+ }
+ }
+}
+
+void
+DefDelWriter_delete_by_query(DefaultDeletionsWriter *self, Query *query)
+{
+ Compiler *compiler = Query_Make_Compiler(query,
+ (Searcher*)self->searcher, Query_Get_Boost(query));
+ uint32_t i, max;
+
+ for (i = 0, max = VA_Get_Size(self->seg_readers); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(self->seg_readers, i);
+ BitVector *bit_vec = (BitVector*)VA_Fetch(self->bit_vecs, i);
+ Matcher *matcher = Compiler_Make_Matcher(compiler, seg_reader, false);
+
+ if (matcher) {
+ int32_t doc_id;
+ int32_t num_zapped = 0;
+
+ // Iterate through matches, marking each doc as deleted.
+ while (0 != (doc_id = Matcher_Next(matcher))) {
+ num_zapped += !BitVec_Get(bit_vec, doc_id);
+ BitVec_Set(bit_vec, doc_id);
+ }
+ if (num_zapped) { self->updated[i] = true; }
+
+ DECREF(matcher);
+ }
+ }
+
+ DECREF(compiler);
+}
+
+void
+DefDelWriter_delete_by_doc_id(DefaultDeletionsWriter *self, int32_t doc_id)
+{
+ uint32_t sub_tick = PolyReader_sub_tick(self->seg_starts, doc_id);
+ BitVector *bit_vec = (BitVector*)VA_Fetch(self->bit_vecs, sub_tick);
+ uint32_t offset = I32Arr_Get(self->seg_starts, sub_tick);
+ int32_t seg_doc_id = doc_id - offset;
+
+ if (!BitVec_Get(bit_vec, seg_doc_id)) {
+ self->updated[sub_tick] = true;
+ BitVec_Set(bit_vec, seg_doc_id);
+ }
+}
+
+bool_t
+DefDelWriter_updated(DefaultDeletionsWriter *self)
+{
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->seg_readers); i < max; i++) {
+ if (self->updated[i]) { return true; }
+ }
+ return false;
+}
+
+void
+DefDelWriter_add_segment(DefaultDeletionsWriter *self, SegReader *reader,
+ I32Array *doc_map)
+{
+ // This method is a no-op, because the only reason it would be called is
+ // if we are adding an entire index. If that's the case, all deletes are
+ // already being applied.
+ UNUSED_VAR(self);
+ UNUSED_VAR(reader);
+ UNUSED_VAR(doc_map);
+}
+
+void
+DefDelWriter_merge_segment(DefaultDeletionsWriter *self, SegReader *reader,
+ I32Array *doc_map)
+{
+ UNUSED_VAR(doc_map);
+ Segment *segment = SegReader_Get_Segment(reader);
+ Hash *del_meta = (Hash*)Seg_Fetch_Metadata_Str(segment, "deletions", 9);
+
+ if (del_meta) {
+ VArray *seg_readers = self->seg_readers;
+ Hash *files = (Hash*)Hash_Fetch_Str(del_meta, "files", 5);
+ if (files) {
+ CharBuf *seg;
+ Hash *mini_meta;
+ Hash_Iterate(files);
+ while(Hash_Next(files, (Obj**)&seg, (Obj**)&mini_meta)) {
+ uint32_t i, max;
+
+ /* Find the segment the deletions from the SegReader
+ * we're adding correspond to. If it's gone, we don't
+ * need to worry about losing deletions files that point
+ * at it. */
+ for (i = 0, max = VA_Get_Size(seg_readers); i < max; i++) {
+ SegReader *candidate
+ = (SegReader*)VA_Fetch(seg_readers, i);
+ CharBuf *candidate_name
+ = Seg_Get_Name(SegReader_Get_Segment(candidate));
+
+ if (CB_Equals(seg, (Obj*)candidate_name)) {
+ /* If the count hasn't changed, we're about to
+ * merge away the most recent deletions file
+ * pointing at this target segment -- so force a
+ * new file to be written out. */
+ int32_t count = (int32_t)Obj_To_I64(
+ Hash_Fetch_Str(mini_meta, "count", 5));
+ DeletionsReader *del_reader
+ = (DeletionsReader*) SegReader_Obtain(candidate,
+ VTable_Get_Name(DELETIONSREADER));
+ if (count == DelReader_Del_Count(del_reader)) {
+ self->updated[i] = true;
+ }
+ break;
+ }
+ }
+ }
+ }
+ }
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Propchange: incubator/lucy/trunk/import/core/KinoSearch/Index/DeletionsWriter.c
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/DeletionsWriter.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/DeletionsWriter.cfh?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/DeletionsWriter.cfh (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/DeletionsWriter.cfh Thu Oct 14 08:48:04 2010
@@ -0,0 +1,155 @@
+parcel KinoSearch cnick Kino;
+
+/** Abstract base class for marking documents as deleted.
+ *
+ * Subclasses of DeletionsWriter provide a low-level mechanism for declaring a
+ * document deleted from an index.
+ *
+ * Because files in an index are never modified, and because it is not
+ * practical to delete entire segments, a DeletionsWriter does not actually
+ * remove documents from the index. Instead, it communicates to a search-time
+ * companion DeletionsReader which documents are deleted in such a way that it
+ * can create a Matcher iterator.
+ *
+ * Documents are truly deleted only when the segments which contain them are
+ * merged into new ones.
+ */
+
+abstract class KinoSearch::Index::DeletionsWriter cnick DelWriter
+ inherits KinoSearch::Index::DataWriter {
+
+ inert DeletionsWriter*
+ init(DeletionsWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader);
+
+ /** Delete all documents in the index that index the supplied term.
+ *
+ * @param field The name of an indexed field. (If it is not spec'd as
+ * <code>indexed</code>, an error will occur.)
+ * @param term The term which identifies docs to be marked as deleted. If
+ * <code>field</code> is associated with an Analyzer, <code>term</code>
+ * will be processed automatically (so don't pre-process it yourself).
+ */
+ public abstract void
+ Delete_By_Term(DeletionsWriter *self, const CharBuf *field, Obj *term);
+
+ /** Delete all documents in the index that match <code>query</code>.
+ *
+ * @param query A L<Query|KinoSearch::Search::Query>.
+ */
+ public abstract void
+ Delete_By_Query(DeletionsWriter *self, Query *query);
+
+ /** Delete the document identified in the PolyReader by the supplied id.
+ */
+ public abstract void
+ Delete_By_Doc_ID(DeletionsWriter *self, int32_t doc_id);
+
+ /** Returns true if there are updates that need to be written.
+ */
+ public abstract bool_t
+ Updated(DeletionsWriter *self);
+
+ /** Produce an array of int32_t which wraps around deleted documents. The
+ * position in the array represents the original doc id, and the value
+ * represents the new doc id. Deleted docs are assigned the value - 0, so
+ * if you had 4 docs and doc 2 was deleted, the array would have the
+ * values... ( 1, 0, 2, 3 ).
+ *
+ * @param offset Value which gets added to each valid document id.
+ * With an offset of 1000, the array in the previous example would be
+ * { 1001, 0, 1002, 1003 }.
+ */
+ public incremented I32Array*
+ Generate_Doc_Map(DeletionsWriter *self, Matcher *deletions,
+ int32_t doc_max, int32_t offset);
+
+ /** Return a deletions iterator for the supplied SegReader, which must be
+ * a component within the PolyReader that was supplied at
+ * construction-time.
+ */
+ public abstract incremented nullable Matcher*
+ Seg_Deletions(DeletionsWriter *self, SegReader *seg_reader);
+
+ /** Return the number of deletions for a given segment.
+ *
+ * @param seg_name The name of the segment.
+ */
+ public abstract int32_t
+ Seg_Del_Count(DeletionsWriter *self, const CharBuf *seg_name);
+}
+
+/** Implements DeletionsWriter using BitVector files.
+ */
+class KinoSearch::Index::DefaultDeletionsWriter cnick DefDelWriter
+ inherits KinoSearch::Index::DeletionsWriter {
+
+ VArray *seg_readers;
+ Hash *name_to_tick;
+ I32Array *seg_starts;
+ VArray *bit_vecs;
+ bool_t *updated;
+ IndexSearcher *searcher;
+
+ inert int32_t current_file_format;
+
+ /**
+ * @param schema A Schema.
+ * @param segment A Segment.
+ * @param snapshot A Snapshot.
+ * @param polyreader An PolyReader.
+ */
+ inert incremented DefaultDeletionsWriter*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader);
+
+ inert DefaultDeletionsWriter*
+ init(DefaultDeletionsWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader);
+
+ public void
+ Delete_By_Term(DefaultDeletionsWriter *self, const CharBuf *field,
+ Obj *term);
+
+ public void
+ Delete_By_Query(DefaultDeletionsWriter *self, Query *query);
+
+ public void
+ Delete_By_Doc_ID(DefaultDeletionsWriter *self, int32_t doc_id);
+
+ public bool_t
+ Updated(DefaultDeletionsWriter *self);
+
+ public incremented nullable Matcher*
+ Seg_Deletions(DefaultDeletionsWriter *self, SegReader *seg_reader);
+
+ public int32_t
+ Seg_Del_Count(DefaultDeletionsWriter *self, const CharBuf *seg_name);
+
+ public void
+ Add_Segment(DefaultDeletionsWriter *self, SegReader *reader,
+ I32Array *doc_map = NULL);
+
+ public void
+ Merge_Segment(DefaultDeletionsWriter *self, SegReader *reader,
+ I32Array *doc_map = NULL);
+
+ public void
+ Finish(DefaultDeletionsWriter *self);
+
+ public int32_t
+ Format(DefaultDeletionsWriter* self);
+
+ public incremented Hash*
+ Metadata(DefaultDeletionsWriter *self);
+
+ public void
+ Destroy(DefaultDeletionsWriter *self);
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/DocReader.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/DocReader.c?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/DocReader.c (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/DocReader.c Thu Oct 14 08:48:04 2010
@@ -0,0 +1,205 @@
+#define C_KINO_DOCREADER
+#define C_KINO_POLYDOCREADER
+#define C_KINO_DEFAULTDOCREADER
+#include "KinoSearch/Util/ToolSet.h"
+
+#include "KinoSearch/Index/DocReader.h"
+#include "KinoSearch/Index/DocWriter.h"
+#include "KinoSearch/Index/PolyReader.h"
+#include "KinoSearch/Index/Segment.h"
+#include "KinoSearch/Index/Snapshot.h"
+#include "KinoSearch/Plan/Schema.h"
+#include "KinoSearch/Store/Folder.h"
+#include "KinoSearch/Store/InStream.h"
+
+DocReader*
+DocReader_init(DocReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick)
+{
+ return (DocReader*)DataReader_init((DataReader*)self, schema, folder,
+ snapshot, segments, seg_tick);
+}
+
+DocReader*
+DocReader_aggregator(DocReader *self, VArray *readers, I32Array *offsets)
+{
+ UNUSED_VAR(self);
+ return (DocReader*)PolyDocReader_new(readers, offsets);
+}
+
+PolyDocReader*
+PolyDocReader_new(VArray *readers, I32Array *offsets)
+{
+ PolyDocReader *self = (PolyDocReader*)VTable_Make_Obj(POLYDOCREADER);
+ return PolyDocReader_init(self, readers, offsets);
+}
+
+PolyDocReader*
+PolyDocReader_init(PolyDocReader *self, VArray *readers, I32Array *offsets)
+{
+ uint32_t i, max;
+ DocReader_init((DocReader*)self, NULL, NULL, NULL, NULL, -1);
+ for (i = 0, max = VA_Get_Size(readers); i < max; i++) {
+ CERTIFY(VA_Fetch(readers, i), DOCREADER);
+ }
+ self->readers = (VArray*)INCREF(readers);
+ self->offsets = (I32Array*)INCREF(offsets);
+ return self;
+}
+
+void
+PolyDocReader_close(PolyDocReader *self)
+{
+ if (self->readers) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->readers); i < max; i++) {
+ DocReader *reader = (DocReader*)VA_Fetch(self->readers, i);
+ if (reader) { DocReader_Close(reader); }
+ }
+ VA_Clear(self->readers);
+ }
+}
+
+void
+PolyDocReader_destroy(PolyDocReader *self)
+{
+ DECREF(self->readers);
+ DECREF(self->offsets);
+ SUPER_DESTROY(self, POLYDOCREADER);
+}
+
+Obj*
+PolyDocReader_fetch(PolyDocReader *self, int32_t doc_id, float score,
+ int32_t offset)
+{
+ uint32_t seg_tick = PolyReader_sub_tick(self->offsets, doc_id);
+ int32_t my_offset = I32Arr_Get(self->offsets, seg_tick);
+ DocReader *doc_reader = (DocReader*)VA_Fetch(self->readers, seg_tick);
+ Obj *hit = NULL;
+ if (!doc_reader) {
+ THROW(ERR, "Invalid doc_id: %i32", doc_id);
+ }
+ else {
+ hit = DocReader_Fetch(doc_reader, doc_id - my_offset, score,
+ offset + my_offset);
+ }
+ return hit;
+}
+
+DefaultDocReader*
+DefDocReader_new(Schema *schema, Folder *folder, Snapshot *snapshot,
+ VArray *segments, int32_t seg_tick)
+{
+ DefaultDocReader *self
+ = (DefaultDocReader*)VTable_Make_Obj(DEFAULTDOCREADER);
+ return DefDocReader_init(self, schema, folder, snapshot, segments,
+ seg_tick);
+}
+
+void
+DefDocReader_close(DefaultDocReader *self)
+{
+ if (self->dat_in != NULL) {
+ InStream_Close(self->dat_in);
+ DECREF(self->dat_in);
+ self->dat_in = NULL;
+ }
+ if (self->ix_in != NULL) {
+ InStream_Close(self->ix_in);
+ DECREF(self->ix_in);
+ self->ix_in = NULL;
+ }
+}
+
+void
+DefDocReader_destroy(DefaultDocReader *self)
+{
+ DECREF(self->ix_in);
+ DECREF(self->dat_in);
+ SUPER_DESTROY(self, DEFAULTDOCREADER);
+}
+
+DefaultDocReader*
+DefDocReader_init(DefaultDocReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick)
+{
+ Hash *metadata;
+ Segment *segment;
+ DocReader_init((DocReader*)self, schema, folder, snapshot, segments,
+ seg_tick);
+ segment = DefDocReader_Get_Segment(self);
+ metadata = (Hash*)Seg_Fetch_Metadata_Str(segment, "documents", 9);
+
+ if (metadata) {
+ CharBuf *seg_name = Seg_Get_Name(segment);
+ CharBuf *ix_file = CB_newf("%o/documents.ix", seg_name);
+ CharBuf *dat_file = CB_newf("%o/documents.dat", seg_name);
+ Obj *format = Hash_Fetch_Str(metadata, "format", 6);
+
+ // Check format.
+ if (!format) { THROW(ERR, "Missing 'format' var"); }
+ else {
+ int64_t format_val = Obj_To_I64(format);
+ if (format_val < DocWriter_current_file_format) {
+ THROW(ERR, "Obsolete doc storage format %i64; "
+ "Index regeneration is required", format_val);
+ }
+ else if (format_val != DocWriter_current_file_format) {
+ THROW(ERR, "Unsupported doc storage format: %i64", format_val);
+ }
+ }
+
+ // Get streams.
+ if (Folder_Exists(folder, ix_file)) {
+ self->ix_in = Folder_Open_In(folder, ix_file);
+ if (!self->ix_in) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(ix_file);
+ DECREF(dat_file);
+ DECREF(self);
+ RETHROW(error);
+ }
+ self->dat_in = Folder_Open_In(folder, dat_file);
+ if (!self->dat_in) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(ix_file);
+ DECREF(dat_file);
+ DECREF(self);
+ RETHROW(error);
+ }
+ }
+ DECREF(ix_file);
+ DECREF(dat_file);
+ }
+
+ return self;
+}
+
+void
+DefDocReader_read_record(DefaultDocReader *self, ByteBuf *buffer,
+ int32_t doc_id)
+{
+ int64_t start;
+ int64_t end;
+ size_t size;
+ char *buf;
+
+ // Find start and length of variable length record.
+ InStream_Seek(self->ix_in, (int64_t)doc_id * 8);
+ start = InStream_Read_I64(self->ix_in);
+ end = InStream_Read_I64(self->ix_in);
+ size = (size_t)(end - start);
+
+ // Read in the record.
+ buf = BB_Grow(buffer, size);
+ InStream_Seek(self->dat_in, start);
+ InStream_Read_Bytes(self->dat_in, buf, size);
+ BB_Set_Size(buffer, size);
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Propchange: incubator/lucy/trunk/import/core/KinoSearch/Index/DocReader.c
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/DocReader.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/DocReader.cfh?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/DocReader.cfh (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/DocReader.cfh Thu Oct 14 08:48:04 2010
@@ -0,0 +1,91 @@
+parcel KinoSearch cnick Kino;
+
+/** Retrieve stored documents.
+ *
+ * DocReader defines the interface by which documents (with all stored fields)
+ * are retrieved from the index. The default implementation returns HitDoc
+ * objects.
+ */
+class KinoSearch::Index::DocReader inherits KinoSearch::Index::DataReader {
+
+ inert DocReader*
+ init(DocReader *self, Schema *schema = NULL, Folder *folder = NULL,
+ Snapshot *snapshot = NULL, VArray *segments = NULL,
+ int32_t seg_tick = -1);
+
+ /** Returns the document identified by <code>doc_id</code>.
+ */
+ public abstract incremented Obj*
+ Fetch(DocReader *self, int32_t doc_id, float score = 0.0, int32_t offset = 0);
+
+ /** Returns a DocReader which divvies up requests to its sub-readers
+ * according to the offset range.
+ *
+ * @param readers An array of DocReaders.
+ * @param offsets Doc id start offsets for each reader.
+ */
+ public incremented nullable DocReader*
+ Aggregator(DocReader *self, VArray *readers, I32Array *offsets);
+}
+
+/** Aggregate multiple DocReaders.
+ */
+class KinoSearch::Index::PolyDocReader inherits KinoSearch::Index::DocReader {
+
+ VArray *readers;
+ I32Array *offsets;
+
+ inert incremented PolyDocReader*
+ new(VArray *readers, I32Array *offsets);
+
+ inert PolyDocReader*
+ init(PolyDocReader *self, VArray *readers, I32Array *offsets);
+
+ public incremented Obj*
+ Fetch(PolyDocReader *self, int32_t doc_id, float score = 0.0,
+ int32_t offset = 0);
+
+ public void
+ Close(PolyDocReader *self);
+
+ public void
+ Destroy(PolyDocReader *self);
+}
+
+class KinoSearch::Index::DefaultDocReader cnick DefDocReader
+ inherits KinoSearch::Index::DocReader {
+
+ InStream *dat_in;
+ InStream *ix_in;
+
+ inert incremented DefaultDocReader*
+ new(Schema *schema, Folder *folder, Snapshot *snapshot, VArray *segments,
+ int32_t seg_tick);
+
+ inert DefaultDocReader*
+ init(DefaultDocReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick);
+
+ public incremented Obj*
+ Fetch(DefaultDocReader *self, int32_t doc_id, float score = 0.0,
+ int32_t offset = 0);
+
+ /** Read the raw byte content for the specified doc into the supplied
+ * buffer.
+ */
+ void
+ Read_Record(DefaultDocReader *self, ByteBuf *buffer, int32_t doc_id);
+
+ public void
+ Close(DefaultDocReader *self);
+
+ public void
+ Destroy(DefaultDocReader *self);
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/DocVector.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/DocVector.c?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/DocVector.c (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/DocVector.c Thu Oct 14 08:48:04 2010
@@ -0,0 +1,193 @@
+#define C_KINO_DOCVECTOR
+#include "KinoSearch/Util/ToolSet.h"
+
+#include "KinoSearch/Index/DocVector.h"
+#include "KinoSearch/Index/TermVector.h"
+#include "KinoSearch/Store/InStream.h"
+#include "KinoSearch/Store/OutStream.h"
+
+// Extract a document's compressed TermVector data into ( term_text =>
+// compressed positional data ) pairs.
+static Hash*
+S_extract_tv_cache(ByteBuf *field_buf);
+
+// Pull a TermVector object out from compressed positional data.
+static TermVector*
+S_extract_tv_from_tv_buf(const CharBuf *field, const CharBuf *term_text,
+ ByteBuf *tv_buf);
+
+DocVector*
+DocVec_new()
+{
+ DocVector *self = (DocVector*)VTable_Make_Obj(DOCVECTOR);
+ return DocVec_init(self);
+}
+
+DocVector*
+DocVec_init(DocVector *self)
+{
+ self->field_bufs = Hash_new(0);
+ self->field_vectors = Hash_new(0);
+ return self;
+}
+
+void
+DocVec_serialize(DocVector *self, OutStream *outstream)
+{
+ Hash_Serialize(self->field_bufs, outstream);
+ Hash_Serialize(self->field_vectors, outstream);
+}
+
+DocVector*
+DocVec_deserialize(DocVector *self, InStream *instream)
+{
+ self = self ? self : (DocVector*)VTable_Make_Obj(DOCVECTOR);
+ self->field_bufs = Hash_deserialize(NULL, instream);
+ self->field_vectors = Hash_deserialize(NULL, instream);
+ return self;
+}
+
+void
+DocVec_destroy(DocVector *self)
+{
+ DECREF(self->field_bufs);
+ DECREF(self->field_vectors);
+ SUPER_DESTROY(self, DOCVECTOR);
+}
+
+void
+DocVec_add_field_buf(DocVector *self, const CharBuf *field,
+ ByteBuf *field_buf)
+{
+ Hash_Store(self->field_bufs, (Obj*)field, INCREF(field_buf));
+}
+
+ByteBuf*
+DocVec_field_buf(DocVector *self, const CharBuf *field)
+{
+ return (ByteBuf*)Hash_Fetch(self->field_bufs, (Obj*)field);
+}
+
+VArray*
+DocVec_field_names(DocVector *self)
+{
+ return Hash_Keys(self->field_bufs);
+}
+
+TermVector*
+DocVec_term_vector(DocVector *self, const CharBuf *field,
+ const CharBuf *term_text)
+{
+ ByteBuf *tv_buf;
+ Hash *field_vector = (Hash*)Hash_Fetch(self->field_vectors, (Obj*)field);
+
+ // If no cache hit, try to fill cache.
+ if (field_vector == NULL) {
+ ByteBuf *field_buf
+ = (ByteBuf*)Hash_Fetch(self->field_bufs, (Obj*)field);
+
+ // Bail if there's no content or the field isn't highlightable.
+ if (field_buf == NULL) return NULL;
+
+ field_vector = S_extract_tv_cache(field_buf);
+ Hash_Store(self->field_vectors, (Obj*)field, (Obj*)field_vector);
+ }
+
+ // Get a buf for the term text or bail.
+ tv_buf = (ByteBuf*)Hash_Fetch(field_vector, (Obj*)term_text);
+ if (tv_buf == NULL)
+ return NULL;
+
+ return S_extract_tv_from_tv_buf(field, term_text, tv_buf);
+}
+
+static Hash*
+S_extract_tv_cache(ByteBuf *field_buf)
+{
+ Hash *tv_cache = Hash_new(0);
+ char *tv_string = BB_Get_Buf(field_buf);
+ int32_t num_terms = NumUtil_decode_c32(&tv_string);
+ CharBuf *text = CB_new(0);
+ int32_t i;
+
+ // Read the number of highlightable terms in the field.
+ for (i = 0; i < num_terms; i++) {
+ char *bookmark_ptr;
+ size_t overlap = NumUtil_decode_c32(&tv_string);
+ size_t len = NumUtil_decode_c32(&tv_string);
+ int32_t num_positions;
+
+ // Decompress the term text.
+ CB_Set_Size(text, overlap);
+ CB_Cat_Trusted_Str(text, tv_string, len);
+ tv_string += len;
+
+ // Get positions & offsets string.
+ bookmark_ptr = tv_string;
+ num_positions = NumUtil_decode_c32(&tv_string);
+ while(num_positions--) {
+ // Leave nums compressed to save a little mem.
+ NumUtil_skip_cint(&tv_string);
+ NumUtil_skip_cint(&tv_string);
+ NumUtil_skip_cint(&tv_string);
+ }
+ len = tv_string - bookmark_ptr;
+
+ // Store the $text => $posdata pair in the output hash.
+ Hash_Store(tv_cache, (Obj*)text,
+ (Obj*)BB_new_bytes(bookmark_ptr, len));
+ }
+ DECREF(text);
+
+ return tv_cache;
+}
+
+static TermVector*
+S_extract_tv_from_tv_buf(const CharBuf *field, const CharBuf *term_text,
+ ByteBuf *tv_buf)
+{
+ TermVector *retval = NULL;
+ char *posdata = BB_Get_Buf(tv_buf);
+ char *posdata_end = posdata + BB_Get_Size(tv_buf);
+ int32_t *positions = NULL;
+ int32_t *starts = NULL;
+ int32_t *ends = NULL;
+ uint32_t num_pos = 0;
+ uint32_t i;
+
+ if (posdata != posdata_end) {
+ num_pos = NumUtil_decode_c32(&posdata);
+ positions = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
+ starts = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
+ ends = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
+ }
+
+ // Expand C32s.
+ for (i = 0; i < num_pos; i++) {
+ positions[i] = NumUtil_decode_c32(&posdata);
+ starts[i] = NumUtil_decode_c32(&posdata);
+ ends[i] = NumUtil_decode_c32(&posdata);
+ }
+
+ if (posdata != posdata_end) {
+ THROW(ERR, "Bad encoding of posdata");
+ }
+ else {
+ I32Array *posits_map = I32Arr_new_steal(positions, num_pos);
+ I32Array *starts_map = I32Arr_new_steal(starts, num_pos);
+ I32Array *ends_map = I32Arr_new_steal(ends, num_pos);
+ retval = TV_new(field, term_text, posits_map, starts_map, ends_map);
+ DECREF(posits_map);
+ DECREF(starts_map);
+ DECREF(ends_map);
+ }
+
+ return retval;
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Propchange: incubator/lucy/trunk/import/core/KinoSearch/Index/DocVector.c
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/DocVector.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/DocVector.cfh?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/DocVector.cfh (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/DocVector.cfh Thu Oct 14 08:48:04 2010
@@ -0,0 +1,49 @@
+parcel KinoSearch cnick Kino;
+
+/** A collection of TermVectors.
+ */
+
+class KinoSearch::Index::DocVector cnick DocVec
+ inherits KinoSearch::Object::Obj {
+
+ Hash *field_bufs;
+ Hash *field_vectors;
+
+ /** Constructor.
+ */
+ inert incremented DocVector*
+ new();
+
+ inert DocVector*
+ init(DocVector *self);
+
+ incremented TermVector*
+ Term_Vector(DocVector *self, const CharBuf *field, const CharBuf *term);
+
+ /** Add a compressed, encoded TermVector to the object.
+ */
+ void
+ Add_Field_Buf(DocVector *self, const CharBuf *field, ByteBuf *field_buf);
+
+ /** Return the compressed, encoded TermVector associated with a particular
+ * field.
+ */
+ ByteBuf*
+ Field_Buf(DocVector *self, const CharBuf *field);
+
+ public void
+ Serialize(DocVector *self, OutStream *outstream);
+
+ public incremented DocVector*
+ Deserialize(DocVector *self, InStream *instream);
+
+ public void
+ Destroy(DocVector *self);
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/DocWriter.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/DocWriter.c?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/DocWriter.c (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/DocWriter.c Thu Oct 14 08:48:04 2010
@@ -0,0 +1,181 @@
+#define C_KINO_DOCWRITER
+#include "KinoSearch/Util/ToolSet.h"
+
+#include "KinoSearch/Index/DocWriter.h"
+#include "KinoSearch/Document/Doc.h"
+#include "KinoSearch/Index/DocReader.h"
+#include "KinoSearch/Index/Inverter.h"
+#include "KinoSearch/Index/PolyReader.h"
+#include "KinoSearch/Index/Segment.h"
+#include "KinoSearch/Index/SegReader.h"
+#include "KinoSearch/Index/Snapshot.h"
+#include "KinoSearch/Plan/FieldType.h"
+#include "KinoSearch/Plan/Schema.h"
+#include "KinoSearch/Store/Folder.h"
+#include "KinoSearch/Store/OutStream.h"
+
+static OutStream*
+S_lazy_init(DocWriter *self);
+
+int32_t DocWriter_current_file_format = 2;
+
+DocWriter*
+DocWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader)
+{
+ DocWriter *self = (DocWriter*)VTable_Make_Obj(DOCWRITER);
+ return DocWriter_init(self, schema, snapshot, segment, polyreader);
+}
+
+DocWriter*
+DocWriter_init(DocWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader)
+{
+ DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
+ return self;
+}
+
+void
+DocWriter_destroy(DocWriter *self)
+{
+ DECREF(self->dat_out);
+ DECREF(self->ix_out);
+ SUPER_DESTROY(self, DOCWRITER);
+}
+
+static OutStream*
+S_lazy_init(DocWriter *self)
+{
+ if (!self->dat_out) {
+ Folder *folder = self->folder;
+ CharBuf *seg_name = Seg_Get_Name(self->segment);
+
+ // Get streams.
+ {
+ CharBuf *ix_file = CB_newf("%o/documents.ix", seg_name);
+ self->ix_out = Folder_Open_Out(folder, ix_file);
+ DECREF(ix_file);
+ if (!self->ix_out) { RETHROW(INCREF(Err_get_error())); }
+ }
+ {
+ CharBuf *dat_file = CB_newf("%o/documents.dat", seg_name);
+ self->dat_out = Folder_Open_Out(folder, dat_file);
+ DECREF(dat_file);
+ if (!self->dat_out) { RETHROW(INCREF(Err_get_error())); }
+ }
+
+ // Go past non-doc #0.
+ OutStream_Write_I64(self->ix_out, 0);
+ }
+
+ return self->dat_out;
+}
+
+void
+DocWriter_add_inverted_doc(DocWriter *self, Inverter *inverter,
+ int32_t doc_id)
+{
+ OutStream *dat_out = S_lazy_init(self);
+ OutStream *ix_out = self->ix_out;
+ uint32_t num_stored = 0;
+ int64_t start = OutStream_Tell(dat_out);
+ int64_t expected = OutStream_Tell(ix_out) / 8;
+
+ // Verify doc id.
+ if (doc_id != expected)
+ THROW(ERR, "Expected doc id %i64 but got %i32", expected, doc_id);
+
+ // Write the number of stored fields.
+ Inverter_Iterate(inverter);
+ while (Inverter_Next(inverter)) {
+ FieldType *type = Inverter_Get_Type(inverter);
+ if (FType_Stored(type)) { num_stored++; }
+ }
+ OutStream_Write_C32(dat_out, num_stored);
+
+ Inverter_Iterate(inverter);
+ while (Inverter_Next(inverter)) {
+ // Only store fields marked as "stored".
+ FieldType *type = Inverter_Get_Type(inverter);
+ if (FType_Stored(type)) {
+ CharBuf *field = Inverter_Get_Field_Name(inverter);
+ Obj *value = Inverter_Get_Value(inverter);
+ CB_Serialize(field, dat_out);
+ Obj_Serialize(value, dat_out);
+ }
+ }
+
+ // Write file pointer.
+ OutStream_Write_I64(ix_out, start);
+}
+
+void
+DocWriter_add_segment(DocWriter *self, SegReader *reader,
+ I32Array *doc_map)
+{
+ int32_t doc_max = SegReader_Doc_Max(reader);
+
+ if (doc_max == 0) {
+ // Bail if the supplied segment is empty.
+ return;
+ }
+ else {
+ OutStream *const dat_out = S_lazy_init(self);
+ OutStream *const ix_out = self->ix_out;
+ ByteBuf *const buffer = BB_new(0);
+ DefaultDocReader *const doc_reader = (DefaultDocReader*)CERTIFY(
+ SegReader_Obtain(reader, VTable_Get_Name(DOCREADER)),
+ DEFAULTDOCREADER);
+ int32_t i, max;
+
+ for (i = 1, max = SegReader_Doc_Max(reader); i <= max; i++) {
+ if (I32Arr_Get(doc_map, i)) {
+ int64_t start = OutStream_Tell(dat_out);
+ char *buf;
+ size_t size;
+
+ // Copy record over.
+ DefDocReader_Read_Record(doc_reader, buffer, i);
+ buf = BB_Get_Buf(buffer);
+ size = BB_Get_Size(buffer);
+ OutStream_Write_Bytes(dat_out, buf, size);
+
+ // Write file pointer.
+ OutStream_Write_I64(ix_out, start);
+ }
+ }
+
+ DECREF(buffer);
+ }
+}
+
+void
+DocWriter_finish(DocWriter *self)
+{
+ if (self->dat_out) {
+ // Write one final file pointer, so that we can derive the length of
+ // the last record.
+ int64_t end = OutStream_Tell(self->dat_out);
+ OutStream_Write_I64(self->ix_out, end);
+
+ // Close down output streams.
+ OutStream_Close(self->dat_out);
+ OutStream_Close(self->ix_out);
+ Seg_Store_Metadata_Str(self->segment, "documents", 9,
+ (Obj*)DocWriter_Metadata(self));
+ }
+}
+
+int32_t
+DocWriter_format(DocWriter *self)
+{
+ UNUSED_VAR(self);
+ return DocWriter_current_file_format;
+}
+
+/* Copyright 2005-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Propchange: incubator/lucy/trunk/import/core/KinoSearch/Index/DocWriter.c
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/DocWriter.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/DocWriter.cfh?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/DocWriter.cfh (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/DocWriter.cfh Thu Oct 14 08:48:04 2010
@@ -0,0 +1,43 @@
+parcel KinoSearch cnick Kino;
+
+/** Default doc writer.
+ */
+class KinoSearch::Index::DocWriter inherits KinoSearch::Index::DataWriter {
+
+ OutStream *ix_out;
+ OutStream *dat_out;
+
+ inert int32_t current_file_format;
+
+ /** Constructors.
+ */
+ inert incremented DocWriter*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader);
+
+ inert DocWriter*
+ init(DocWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader);
+
+ public void
+ Add_Inverted_Doc(DocWriter *self, Inverter *inverter, int32_t doc_id);
+
+ public void
+ Add_Segment(DocWriter *self, SegReader *reader, I32Array *doc_map = NULL);
+
+ public void
+ Finish(DocWriter *self);
+
+ public int32_t
+ Format(DocWriter *self);
+
+ public void
+ Destroy(DocWriter *self);
+}
+
+/* Copyright 2005-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/FilePurger.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/FilePurger.c?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/FilePurger.c (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/FilePurger.c Thu Oct 14 08:48:04 2010
@@ -0,0 +1,280 @@
+#define C_KINO_FILEPURGER
+#include <ctype.h>
+#include "KinoSearch/Util/ToolSet.h"
+
+#include "KinoSearch/Index/FilePurger.h"
+#include "KinoSearch/Index/IndexManager.h"
+#include "KinoSearch/Index/Segment.h"
+#include "KinoSearch/Index/Snapshot.h"
+#include "KinoSearch/Plan/Schema.h"
+#include "KinoSearch/Store/DirHandle.h"
+#include "KinoSearch/Store/Folder.h"
+#include "KinoSearch/Store/Lock.h"
+
+// Place unused files into purgables array and obsolete Snapshots into
+// snapshots array.
+static void
+S_discover_unused(FilePurger *self, VArray **purgables, VArray **snapshots);
+
+// Clean up after a failed background merge session, adding all dead files to
+// the list of candidates to be zapped.
+static void
+S_zap_dead_merge(FilePurger *self, Hash *candidates);
+
+// Return an array of recursively expanded filepath entries.
+static VArray*
+S_find_all_referenced(Folder *folder, VArray *entries);
+
+FilePurger*
+FilePurger_new(Folder *folder, Snapshot *snapshot, IndexManager *manager)
+{
+ FilePurger *self = (FilePurger*)VTable_Make_Obj(FILEPURGER);
+ return FilePurger_init(self, folder, snapshot, manager);
+}
+
+FilePurger*
+FilePurger_init(FilePurger *self, Folder *folder, Snapshot *snapshot,
+ IndexManager *manager)
+{
+ self->folder = (Folder*)INCREF(folder);
+ self->snapshot = (Snapshot*)INCREF(snapshot);
+ self->manager = manager
+ ? (IndexManager*)INCREF(manager)
+ : IxManager_new(NULL, NULL);
+ IxManager_Set_Folder(self->manager, folder);
+
+ // Don't allow the locks directory to be zapped.
+ self->disallowed = Hash_new(0);
+ Hash_Store_Str(self->disallowed, "locks", 5, INCREF(&EMPTY));
+
+ return self;
+}
+
+void
+FilePurger_destroy(FilePurger *self)
+{
+ DECREF(self->folder);
+ DECREF(self->snapshot);
+ DECREF(self->manager);
+ DECREF(self->disallowed);
+ SUPER_DESTROY(self, FILEPURGER);
+}
+
+void
+FilePurger_purge(FilePurger *self)
+{
+ Lock *deletion_lock = IxManager_Make_Deletion_Lock(self->manager);
+
+ // Obtain deletion lock, purge files, release deletion lock.
+ Lock_Clear_Stale(deletion_lock);
+ if (Lock_Obtain(deletion_lock)) {
+ Folder *folder = self->folder;
+ Hash *failures = Hash_new(0);
+ VArray *purgables;
+ VArray *snapshots;
+
+ S_discover_unused(self, &purgables, &snapshots);
+
+ // Attempt to delete entries -- if failure, no big deal, just try
+ // again later. Proceed in reverse lexical order so that directories
+ // get deleted after they've been emptied.
+ VA_Sort(purgables, NULL, NULL);
+ for (uint32_t i = VA_Get_Size(purgables); i--; ) {
+ CharBuf *entry = (CharBuf*)VA_fetch(purgables, i);
+ if (Hash_Fetch(self->disallowed, (Obj*)entry)) { continue; }
+ if (!Folder_Delete(folder, entry)) {
+ if (Folder_Exists(folder, entry)) {
+ Hash_Store(failures, (Obj*)entry, INCREF(&EMPTY));
+ }
+ }
+ }
+
+ for (uint32_t i = 0, max = VA_Get_Size(snapshots); i < max; i++) {
+ Snapshot *snapshot = (Snapshot*)VA_Fetch(snapshots, i);
+ bool_t snapshot_has_failures = false;
+ if (Hash_Get_Size(failures)) {
+ // Only delete snapshot files if all of their entries were
+ // successfully deleted.
+ VArray *entries = Snapshot_List(snapshot);
+ for (uint32_t j = VA_Get_Size(entries); j--; ) {
+ CharBuf *entry = (CharBuf*)VA_Fetch(entries, j);
+ if (Hash_Fetch(failures, (Obj*)entry)) {
+ snapshot_has_failures = true;
+ break;
+ }
+ }
+ DECREF(entries);
+ }
+ if (!snapshot_has_failures) {
+ CharBuf *snapfile = Snapshot_Get_Path(snapshot);
+ Folder_Delete(folder, snapfile);
+ }
+ }
+
+ DECREF(failures);
+ DECREF(purgables);
+ DECREF(snapshots);
+ Lock_Release(deletion_lock);
+ }
+ else {
+ WARN("Can't obtain deletion lock, skipping deletion of "
+ "obsolete files");
+ }
+
+ DECREF(deletion_lock);
+}
+
+static void
+S_zap_dead_merge(FilePurger *self, Hash *candidates)
+{
+ IndexManager *manager = self->manager;
+ Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
+
+ Lock_Clear_Stale(merge_lock);
+ if (!Lock_Is_Locked(merge_lock)) {
+ Hash *merge_data = IxManager_Read_Merge_Data(manager);
+ Obj *cutoff = merge_data
+ ? Hash_Fetch_Str(merge_data, "cutoff", 6)
+ : NULL;
+
+ if (cutoff) {
+ CharBuf *cutoff_seg = Seg_num_to_name(Obj_To_I64(cutoff));
+ if (Folder_Exists(self->folder, cutoff_seg)) {
+ ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
+ DirHandle *dh = Folder_Open_Dir(self->folder, cutoff_seg);
+ CharBuf *entry = dh ? DH_Get_Entry(dh) : NULL;
+ CharBuf *filepath = CB_new(32);
+
+ if (!dh) {
+ THROW(ERR, "Can't open segment dir '%o'", filepath);
+ }
+
+ Hash_Store(candidates, (Obj*)cutoff_seg, INCREF(&EMPTY));
+ Hash_Store(candidates, (Obj*)merge_json, INCREF(&EMPTY));
+ while (DH_Next(dh)) {
+ // TODO: recursively delete subdirs within seg dir.
+ CB_setf(filepath, "%o/%o", cutoff_seg, entry);
+ Hash_Store(candidates, (Obj*)filepath, INCREF(&EMPTY));
+ }
+ DECREF(filepath);
+ DECREF(dh);
+ }
+ DECREF(cutoff_seg);
+ }
+
+ DECREF(merge_data);
+ }
+
+ DECREF(merge_lock);
+ return;
+}
+
+static void
+S_discover_unused(FilePurger *self, VArray **purgables_ptr,
+ VArray **snapshots_ptr)
+{
+ Folder *folder = self->folder;
+ DirHandle *dh = Folder_Open_Dir(folder, NULL);
+ if (!dh) { RETHROW(INCREF(Err_get_error())); }
+ VArray *spared = VA_new(1);
+ VArray *snapshots = VA_new(1);
+ CharBuf *snapfile = NULL;
+
+ // Start off with the list of files in the current snapshot.
+ if (self->snapshot) {
+ VArray *entries = Snapshot_List(self->snapshot);
+ VArray *referenced = S_find_all_referenced(folder, entries);
+ VA_Push_VArray(spared, referenced);
+ DECREF(entries);
+ DECREF(referenced);
+ snapfile = Snapshot_Get_Path(self->snapshot);
+ if (snapfile) { VA_Push(spared, INCREF(snapfile)); }
+ }
+
+ CharBuf *entry = DH_Get_Entry(dh);
+ Hash *candidates = Hash_new(64);
+ while (DH_Next(dh)) {
+ if (!CB_Starts_With_Str(entry, "snapshot_", 9)) { continue; }
+ else if (!CB_Ends_With_Str(entry, ".json", 5)) { continue; }
+ else if (snapfile && CB_Equals(entry, (Obj*)snapfile)) { continue; }
+ else {
+ Snapshot *snapshot
+ = Snapshot_Read_File(Snapshot_new(), folder, entry);
+ Lock *lock
+ = IxManager_Make_Snapshot_Read_Lock(self->manager, entry);
+ VArray *snap_list = Snapshot_List(snapshot);
+ VArray *referenced = S_find_all_referenced(folder, snap_list);
+
+ // DON'T obtain the lock -- only see whether another
+ // entity holds a lock on the snapshot file.
+ if (lock && Lock_Is_Locked(lock)) {
+ // The snapshot file is locked, which means someone's using
+ // that version of the index -- protect all of its entries.
+ uint32_t new_size = VA_Get_Size(spared)
+ + VA_Get_Size(referenced) + 1;
+ VA_Grow(spared, new_size);
+ VA_Push(spared, (Obj*)CB_Clone(entry));
+ VA_Push_VArray(spared, referenced);
+ }
+ else {
+ // No one's using this snapshot, so all of its entries are
+ // candidates for deletion.
+ for (uint32_t i = 0, max = VA_Get_Size(referenced); i < max; i++) {
+ CharBuf *file = (CharBuf*)VA_Fetch(referenced, i);
+ Hash_Store(candidates, (Obj*)file, INCREF(&EMPTY));
+ }
+ VA_Push(snapshots, INCREF(snapshot));
+ }
+
+ DECREF(referenced);
+ DECREF(snap_list);
+ DECREF(snapshot);
+ DECREF(lock);
+ }
+ }
+ DECREF(dh);
+
+ // Clean up after a dead segment consolidation.
+ S_zap_dead_merge(self, candidates);
+
+ // Eliminate any current files from the list of files to be purged.
+ for (uint32_t i = 0, max = VA_Get_Size(spared); i < max; i++) {
+ CharBuf *filename = (CharBuf*)VA_Fetch(spared, i);
+ DECREF(Hash_Delete(candidates, (Obj*)filename));
+ }
+
+ // Pass back purgables and Snapshots.
+ *purgables_ptr = Hash_Keys(candidates);
+ *snapshots_ptr = snapshots;
+
+ DECREF(candidates);
+ DECREF(spared);
+}
+
+static VArray*
+S_find_all_referenced(Folder *folder, VArray *entries)
+{
+ Hash *uniqued = Hash_new(VA_Get_Size(entries));
+ for (uint32_t i = 0, max = VA_Get_Size(entries); i < max; i++) {
+ CharBuf *entry = (CharBuf*)VA_Fetch(entries, i);
+ Hash_Store(uniqued, (Obj*)entry, INCREF(&EMPTY));
+ if (Folder_Is_Directory(folder, entry)) {
+ VArray *contents = Folder_List_R(folder, entry);
+ for (uint32_t j = VA_Get_Size(contents); j--; ) {
+ CharBuf *sub_entry = (CharBuf*)VA_Fetch(contents, j);
+ Hash_Store(uniqued, (Obj*)sub_entry, INCREF(&EMPTY));
+ }
+ DECREF(contents);
+ }
+ }
+ VArray *referenced = Hash_Keys(uniqued);
+ DECREF(uniqued);
+ return referenced;
+}
+
+/* Copyright 2007-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Propchange: incubator/lucy/trunk/import/core/KinoSearch/Index/FilePurger.c
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/FilePurger.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/FilePurger.cfh?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/FilePurger.cfh (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/FilePurger.cfh Thu Oct 14 08:48:04 2010
@@ -0,0 +1,35 @@
+parcel KinoSearch cnick Kino;
+
+/** Delete obsolete index files.
+ */
+
+class KinoSearch::Index::FilePurger inherits KinoSearch::Object::Obj {
+
+ Folder *folder;
+ Snapshot *snapshot;
+ IndexManager *manager;
+ Hash *disallowed;
+
+ inert incremented FilePurger*
+ new(Folder *folder, Snapshot *snapshot = NULL,
+ IndexManager *manager = NULL);
+
+ inert FilePurger*
+ init(FilePurger *self, Folder *folder, Snapshot *snapshot = NULL,
+ IndexManager *manager = NULL);
+
+ /** Purge obsolete files from the index.
+ */
+ void
+ Purge(FilePurger *self);
+
+ public void
+ Destroy(FilePurger *self);
+}
+
+/* Copyright 2007-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightReader.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightReader.c?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightReader.c (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightReader.c Thu Oct 14 08:48:04 2010
@@ -0,0 +1,233 @@
+#define C_KINO_HIGHLIGHTREADER
+#define C_KINO_POLYHIGHLIGHTREADER
+#define C_KINO_DEFAULTHIGHLIGHTREADER
+#include "KinoSearch/Util/ToolSet.h"
+
+#include "KinoSearch/Index/HighlightReader.h"
+#include "KinoSearch/Index/DocVector.h"
+#include "KinoSearch/Index/HighlightWriter.h"
+#include "KinoSearch/Index/PolyReader.h"
+#include "KinoSearch/Index/Segment.h"
+#include "KinoSearch/Index/Snapshot.h"
+#include "KinoSearch/Plan/Schema.h"
+#include "KinoSearch/Store/InStream.h"
+#include "KinoSearch/Store/OutStream.h"
+#include "KinoSearch/Store/Folder.h"
+
+HighlightReader*
+HLReader_init(HighlightReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick)
+{
+ DataReader_init((DataReader*)self, schema, folder, snapshot, segments,
+ seg_tick);
+ ABSTRACT_CLASS_CHECK(self, HIGHLIGHTREADER);
+ return self;
+}
+
+HighlightReader*
+HLReader_aggregator(HighlightReader *self, VArray *readers,
+ I32Array *offsets)
+{
+ UNUSED_VAR(self);
+ return (HighlightReader*)PolyHLReader_new(readers, offsets);
+}
+
+PolyHighlightReader*
+PolyHLReader_new(VArray *readers, I32Array *offsets)
+{
+ PolyHighlightReader *self
+ = (PolyHighlightReader*)VTable_Make_Obj(POLYHIGHLIGHTREADER);
+ return PolyHLReader_init(self, readers, offsets);
+}
+
+PolyHighlightReader*
+PolyHLReader_init(PolyHighlightReader *self, VArray *readers,
+ I32Array *offsets)
+{
+ uint32_t i, max;
+ HLReader_init((HighlightReader*)self, NULL, NULL, NULL, NULL, -1);
+ for (i = 0, max = VA_Get_Size(readers); i < max; i++) {
+ CERTIFY(VA_Fetch(readers, i), HIGHLIGHTREADER);
+ }
+ self->readers = (VArray*)INCREF(readers);
+ self->offsets = (I32Array*)INCREF(offsets);
+ return self;
+}
+
+void
+PolyHLReader_close(PolyHighlightReader *self)
+{
+ if (self->readers) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->readers); i < max; i++) {
+ HighlightReader *sub_reader
+ = (HighlightReader*)VA_Fetch(self->readers, i);
+ if (sub_reader) { HLReader_Close(sub_reader); }
+ }
+ DECREF(self->readers);
+ DECREF(self->offsets);
+ self->readers = NULL;
+ self->offsets = NULL;
+ }
+}
+
+void
+PolyHLReader_destroy(PolyHighlightReader *self)
+{
+ DECREF(self->readers);
+ DECREF(self->offsets);
+ SUPER_DESTROY(self, POLYHIGHLIGHTREADER);
+}
+
+DocVector*
+PolyHLReader_fetch(PolyHighlightReader *self, int32_t doc_id)
+{
+ uint32_t seg_tick = PolyReader_sub_tick(self->offsets, doc_id);
+ int32_t offset = I32Arr_Get(self->offsets, seg_tick);
+ HighlightReader *sub_reader
+ = (HighlightReader*)VA_Fetch(self->readers, seg_tick);
+ if (!sub_reader) { THROW(ERR, "Invalid doc_id: %i32", doc_id); }
+ return HLReader_Fetch(sub_reader, doc_id - offset);
+}
+
+DefaultHighlightReader*
+DefHLReader_new(Schema *schema, Folder *folder, Snapshot *snapshot,
+ VArray *segments, int32_t seg_tick)
+{
+ DefaultHighlightReader *self = (DefaultHighlightReader*)
+ VTable_Make_Obj(DEFAULTHIGHLIGHTREADER);
+ return DefHLReader_init(self, schema, folder, snapshot, segments,
+ seg_tick);
+}
+
+DefaultHighlightReader*
+DefHLReader_init(DefaultHighlightReader *self, Schema *schema,
+ Folder *folder, Snapshot *snapshot, VArray *segments,
+ int32_t seg_tick)
+{
+ Segment *segment;
+ Hash *metadata;
+ HLReader_init((HighlightReader*)self, schema, folder, snapshot,
+ segments, seg_tick);
+ segment = DefHLReader_Get_Segment(self);
+ metadata = (Hash*)Seg_Fetch_Metadata_Str(segment, "highlight", 9);
+ if (!metadata) {
+ metadata = (Hash*)Seg_Fetch_Metadata_Str(segment, "term_vectors", 12);
+ }
+
+ // Check format.
+ if (metadata) {
+ Obj *format = Hash_Fetch_Str(metadata, "format", 6);
+ if (!format) { THROW(ERR, "Missing 'format' var"); }
+ else {
+ if (Obj_To_I64(format) != HLWriter_current_file_format) {
+ THROW(ERR, "Unsupported highlight data format: %i64",
+ Obj_To_I64(format));
+ }
+ }
+ }
+
+
+ // Open instreams.
+ {
+ CharBuf *seg_name = Seg_Get_Name(segment);
+ CharBuf *ix_file = CB_newf("%o/highlight.ix", seg_name);
+ CharBuf *dat_file = CB_newf("%o/highlight.dat", seg_name);
+ if (Folder_Exists(folder, ix_file)) {
+ self->ix_in = Folder_Open_In(folder, ix_file);
+ if (!self->ix_in) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(ix_file);
+ DECREF(dat_file);
+ DECREF(self);
+ RETHROW(error);
+ }
+ self->dat_in = Folder_Open_In(folder, dat_file);
+ if (!self->dat_in) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(ix_file);
+ DECREF(dat_file);
+ DECREF(self);
+ RETHROW(error);
+ }
+ }
+ DECREF(ix_file);
+ DECREF(dat_file);
+ }
+
+ return self;
+}
+
+void
+DefHLReader_close(DefaultHighlightReader *self)
+{
+ if (self->dat_in != NULL) {
+ InStream_Close(self->dat_in);
+ DECREF(self->dat_in);
+ self->dat_in = NULL;
+ }
+ if (self->ix_in != NULL) {
+ InStream_Close(self->ix_in);
+ DECREF(self->ix_in);
+ self->ix_in = NULL;
+ }
+}
+
+void
+DefHLReader_destroy(DefaultHighlightReader *self)
+{
+ DECREF(self->ix_in);
+ DECREF(self->dat_in);
+ SUPER_DESTROY(self, DEFAULTHIGHLIGHTREADER);
+}
+
+DocVector*
+DefHLReader_fetch(DefaultHighlightReader *self, int32_t doc_id)
+{
+ DocVector *doc_vec = DocVec_new();
+ int64_t file_pos;
+ uint32_t num_fields;
+
+ InStream_Seek(self->ix_in, doc_id * 8);
+ file_pos = InStream_Read_I64(self->ix_in);
+ InStream_Seek(self->dat_in, file_pos);
+
+ num_fields = InStream_Read_C32(self->dat_in);
+ while (num_fields--) {
+ CharBuf *field = CB_deserialize(NULL, self->dat_in);
+ ByteBuf *field_buf = BB_deserialize(NULL, self->dat_in);
+ DocVec_Add_Field_Buf(doc_vec, field, field_buf);
+ DECREF(field_buf);
+ DECREF(field);
+ }
+
+ return doc_vec;
+}
+
+void
+DefHLReader_read_record(DefaultHighlightReader *self, int32_t doc_id,
+ ByteBuf *target)
+{
+ InStream *dat_in = self->dat_in;
+ InStream *ix_in = self->ix_in;
+
+ InStream_Seek(ix_in, doc_id * 8);
+
+ {
+ // Copy the whole record.
+ int64_t filepos = InStream_Read_I64(ix_in);
+ int64_t end = InStream_Read_I64(ix_in);
+ size_t size = (size_t)(end - filepos);
+ char *buf = BB_Grow(target, size);
+ InStream_Seek(dat_in, filepos);
+ InStream_Read_Bytes(dat_in, buf, size);
+ BB_Set_Size(target, size);
+ }
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Propchange: incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightReader.c
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightReader.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightReader.cfh?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightReader.cfh (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightReader.cfh Thu Oct 14 08:48:04 2010
@@ -0,0 +1,85 @@
+parcel KinoSearch cnick Kino;
+
+/** Read a segment's highlighting data.
+ *
+ * Read segment data used for creating highlighted excerpts.
+ */
+class KinoSearch::Index::HighlightReader cnick HLReader
+ inherits KinoSearch::Index::DataReader {
+
+ inert HighlightReader*
+ init(HighlightReader *self, Schema *schema = NULL, Folder *folder = NULL,
+ Snapshot *snapshot = NULL, VArray *segments = NULL,
+ int32_t seg_tick = -1);
+
+ /** Return the DocVector object for the given doc id.
+ */
+ public abstract incremented DocVector*
+ Fetch(HighlightReader *self, int32_t doc_id);
+
+ public incremented nullable HighlightReader*
+ Aggregator(HighlightReader *self, VArray *readers, I32Array *offsets);
+}
+
+class KinoSearch::Index::PolyHighlightReader cnick PolyHLReader
+ inherits KinoSearch::Index::HighlightReader {
+
+ VArray *readers;
+ I32Array *offsets;
+
+ inert incremented PolyHighlightReader*
+ new(VArray *readers, I32Array *offsets);
+
+ inert PolyHighlightReader*
+ init(PolyHighlightReader *self, VArray *readers, I32Array *offsets);
+
+ public incremented DocVector*
+ Fetch(PolyHighlightReader *self, int32_t doc_id);
+
+ public void
+ Close(PolyHighlightReader *self);
+
+ public void
+ Destroy(PolyHighlightReader *self);
+}
+
+class KinoSearch::Index::DefaultHighlightReader cnick DefHLReader
+ inherits KinoSearch::Index::HighlightReader {
+
+ InStream *ix_in;
+ InStream *dat_in;
+
+ /** Constructors.
+ */
+ inert incremented DefaultHighlightReader*
+ new(Schema *schema, Folder *folder, Snapshot *snapshot, VArray *segments,
+ int32_t seg_tick);
+
+ inert DefaultHighlightReader*
+ init(DefaultHighlightReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick);
+
+ /** Return the DocVector object for the given doc id.
+ */
+ public incremented DocVector*
+ Fetch(DefaultHighlightReader *self, int32_t doc_id);
+
+ /** Return the raw bytes of an entry.
+ */
+ void
+ Read_Record(DefaultHighlightReader *self, int32_t doc_id,
+ ByteBuf *buffer);
+
+ public void
+ Close(DefaultHighlightReader *self);
+
+ public void
+ Destroy(DefaultHighlightReader *self);
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightWriter.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightWriter.c?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightWriter.c (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightWriter.c Thu Oct 14 08:48:04 2010
@@ -0,0 +1,264 @@
+#define C_KINO_HIGHLIGHTWRITER
+#define C_KINO_DEFAULTHIGHLIGHTWRITER
+#define C_KINO_TOKEN
+#include "KinoSearch/Util/ToolSet.h"
+
+#include <stdio.h>
+
+#include "KinoSearch/Index/HighlightWriter.h"
+#include "KinoSearch/Analysis/Token.h"
+#include "KinoSearch/Analysis/Inversion.h"
+#include "KinoSearch/Plan/FieldType.h"
+#include "KinoSearch/Plan/FullTextType.h"
+#include "KinoSearch/Index/HighlightReader.h"
+#include "KinoSearch/Index/Inverter.h"
+#include "KinoSearch/Index/PolyReader.h"
+#include "KinoSearch/Index/SegReader.h"
+#include "KinoSearch/Index/Segment.h"
+#include "KinoSearch/Index/Snapshot.h"
+#include "KinoSearch/Plan/Schema.h"
+#include "KinoSearch/Store/Folder.h"
+#include "KinoSearch/Store/OutStream.h"
+#include "KinoSearch/Store/InStream.h"
+
+static OutStream*
+S_lazy_init(HighlightWriter *self);
+
+int32_t HLWriter_current_file_format = 1;
+
+HighlightWriter*
+HLWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader)
+{
+ HighlightWriter *self
+ = (HighlightWriter*)VTable_Make_Obj(HIGHLIGHTWRITER);
+ return HLWriter_init(self, schema, snapshot, segment, polyreader);
+}
+
+HighlightWriter*
+HLWriter_init(HighlightWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader)
+{
+ DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
+ return self;
+}
+
+void
+HLWriter_destroy(HighlightWriter *self)
+{
+ DECREF(self->dat_out);
+ DECREF(self->ix_out);
+ SUPER_DESTROY(self, HIGHLIGHTWRITER);
+}
+
+static OutStream*
+S_lazy_init(HighlightWriter *self)
+{
+ if (!self->dat_out) {
+ Segment *segment = self->segment;
+ Folder *folder = self->folder;
+ CharBuf *seg_name = Seg_Get_Name(segment);
+
+ // Open outstreams.
+ {
+ CharBuf *ix_file = CB_newf("%o/highlight.ix", seg_name);
+ self->ix_out = Folder_Open_Out(folder, ix_file);
+ DECREF(ix_file);
+ if (!self->ix_out) { RETHROW(INCREF(Err_get_error())); }
+ }
+ {
+ CharBuf *dat_file = CB_newf("%o/highlight.dat", seg_name);
+ self->dat_out = Folder_Open_Out(folder, dat_file);
+ DECREF(dat_file);
+ if (!self->dat_out) { RETHROW(INCREF(Err_get_error())); }
+ }
+
+ // Go past invalid doc 0.
+ OutStream_Write_I64(self->ix_out, 0);
+ }
+
+ return self->dat_out;
+}
+
+void
+HLWriter_add_inverted_doc(HighlightWriter *self, Inverter *inverter,
+ int32_t doc_id)
+{
+ OutStream *dat_out = S_lazy_init(self);
+ OutStream *ix_out = self->ix_out;
+ int64_t filepos = OutStream_Tell(dat_out);
+ uint32_t num_highlightable = 0;
+ int32_t expected = (int32_t)(OutStream_Tell(ix_out) / 8);
+
+ // Verify doc id.
+ if (doc_id != expected)
+ THROW(ERR, "Expected doc id %i32 but got %i32", expected, doc_id);
+
+ // Write index data.
+ OutStream_Write_I64(ix_out, filepos);
+
+ // Count, then write number of highlightable fields.
+ Inverter_Iterate(inverter);
+ while (Inverter_Next(inverter)) {
+ FieldType *type = Inverter_Get_Type(inverter);
+ if ( FType_Is_A(type, FULLTEXTTYPE)
+ && FullTextType_Highlightable((FullTextType*)type)
+ ) {
+ num_highlightable++;
+ }
+ }
+ OutStream_Write_C32(dat_out, num_highlightable);
+
+ Inverter_Iterate(inverter);
+ while (Inverter_Next(inverter)) {
+ FieldType *type = Inverter_Get_Type(inverter);
+ if ( FType_Is_A(type, FULLTEXTTYPE)
+ && FullTextType_Highlightable((FullTextType*)type)
+ ) {
+ CharBuf *field = Inverter_Get_Field_Name(inverter);
+ Inversion *inversion = Inverter_Get_Inversion(inverter);
+ ByteBuf *tv_buf = HLWriter_TV_Buf(self, inversion);
+ CB_Serialize(field, dat_out);
+ BB_Serialize(tv_buf, dat_out);
+ DECREF(tv_buf);
+ }
+ }
+}
+
+ByteBuf*
+HLWriter_tv_buf(HighlightWriter *self, Inversion *inversion)
+{
+ char *last_text = "";
+ size_t last_len = 0;
+ ByteBuf *tv_buf = BB_new(20 + Inversion_Get_Size(inversion) * 8);
+ uint32_t num_postings = 0;
+ char *dest;
+ Token **tokens;
+ uint32_t freq;
+ UNUSED_VAR(self);
+
+ // Leave space for a c32 indicating the number of postings.
+ BB_Set_Size(tv_buf, C32_MAX_BYTES);
+
+ Inversion_Reset(inversion);
+ while ( (tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL ) {
+ Token *token = *tokens;
+ int32_t overlap = StrHelp_overlap(last_text, token->text,
+ last_len, token->len);
+ char *ptr;
+ char *orig;
+ size_t old_size = BB_Get_Size(tv_buf);
+ size_t new_size = old_size
+ + C32_MAX_BYTES // overlap
+ + C32_MAX_BYTES // length of string diff
+ + (token->len - overlap) // diff char data
+ + C32_MAX_BYTES // num prox
+ + (C32_MAX_BYTES * freq * 3); // pos data
+
+ // Allocate for worst-case scenario.
+ ptr = BB_Grow(tv_buf, new_size);
+ orig = ptr;
+ ptr += old_size;
+
+ // Track number of postings.
+ num_postings += 1;
+
+ // Append the string diff to the tv_buf.
+ NumUtil_encode_c32(overlap, &ptr);
+ NumUtil_encode_c32( (token->len - overlap), &ptr);
+ memcpy(ptr, (token->text + overlap), (token->len - overlap));
+ ptr += token->len - overlap;
+
+ // Save text and text_len for comparison next loop.
+ last_text = token->text;
+ last_len = token->len;
+
+ // Append the number of positions for this term.
+ NumUtil_encode_c32(freq, &ptr);
+
+ do {
+ // Add position, start_offset, and end_offset to tv_buf.
+ NumUtil_encode_c32(token->pos, &ptr);
+ NumUtil_encode_c32(token->start_offset, &ptr);
+ NumUtil_encode_c32(token->end_offset, &ptr);
+
+ } while (--freq && (token = *++tokens));
+
+ // Set new byte length.
+ BB_Set_Size(tv_buf, ptr - orig);
+ }
+
+ // Go back and start the term vector string with the posting count.
+ dest = BB_Get_Buf(tv_buf);
+ NumUtil_encode_padded_c32(num_postings, &dest);
+
+ return tv_buf;
+}
+
+void
+HLWriter_add_segment(HighlightWriter *self, SegReader *reader,
+ I32Array *doc_map)
+{
+ int32_t doc_max = SegReader_Doc_Max(reader);
+
+ if (doc_max == 0) {
+ // Bail if the supplied segment is empty.
+ return;
+ }
+ else {
+ DefaultHighlightReader *hl_reader = (DefaultHighlightReader*)
+ CERTIFY(SegReader_Obtain(reader,
+ VTable_Get_Name(HIGHLIGHTREADER)), DEFAULTHIGHLIGHTREADER);
+ OutStream *dat_out = S_lazy_init(self);
+ OutStream *ix_out = self->ix_out;
+ int32_t orig;
+ ByteBuf *bb = BB_new(0);
+
+ for (orig = 1; orig <= doc_max; orig++) {
+ // Skip deleted docs.
+ if (doc_map && !I32Arr_Get(doc_map, orig))
+ continue;
+
+ // Write file pointer.
+ OutStream_Write_I64( ix_out, OutStream_Tell(dat_out) );
+
+ // Copy the raw record.
+ DefHLReader_Read_Record(hl_reader, orig, bb);
+ OutStream_Write_Bytes(dat_out, BB_Get_Buf(bb), BB_Get_Size(bb));
+
+ BB_Set_Size(bb, 0);
+ }
+ DECREF(bb);
+ }
+}
+
+void
+HLWriter_finish(HighlightWriter *self)
+{
+ if (self->dat_out) {
+ // Write one final file pointer, so that we can derive the length of
+ // the last record.
+ int64_t end = OutStream_Tell(self->dat_out);
+ OutStream_Write_I64(self->ix_out, end);
+
+ // Close down the output streams.
+ OutStream_Close(self->dat_out);
+ OutStream_Close(self->ix_out);
+ Seg_Store_Metadata_Str(self->segment, "highlight", 9,
+ (Obj*)HLWriter_Metadata(self));
+ }
+}
+
+int32_t
+HLWriter_format(HighlightWriter *self)
+{
+ UNUSED_VAR(self);
+ return HLWriter_current_file_format;
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Propchange: incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightWriter.c
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightWriter.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightWriter.cfh?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightWriter.cfh (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Index/HighlightWriter.cfh Thu Oct 14 08:48:04 2010
@@ -0,0 +1,47 @@
+parcel KinoSearch cnick Kino;
+
+/** Add highlighting data to index.
+ */
+
+class KinoSearch::Index::HighlightWriter cnick HLWriter
+ inherits KinoSearch::Index::DataWriter {
+
+ OutStream *ix_out;
+ OutStream *dat_out;
+
+ inert int32_t current_file_format;
+
+ inert incremented HighlightWriter*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader);
+
+ inert HighlightWriter*
+ init(HighlightWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader);
+
+ incremented ByteBuf*
+ TV_Buf(HighlightWriter *self, Inversion *inversion);
+
+ public void
+ Add_Inverted_Doc(HighlightWriter *self, Inverter *inverter, int32_t doc_id);
+
+ public void
+ Add_Segment(HighlightWriter *self, SegReader *reader,
+ I32Array *doc_map = NULL);
+
+ public void
+ Finish(HighlightWriter *self);
+
+ public int32_t
+ Format(HighlightWriter *self);
+
+ public void
+ Destroy(HighlightWriter *self);
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+