You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by mi...@apache.org on 2010/10/14 10:48:40 UTC
[lucy-commits] svn commit: r1022422 [23/43] - in /incubator/lucy/trunk/import: ./
charmonizer/ charmonizer/src/ charmonizer/src/Charmonizer/
charmonizer/src/Charmonizer/Core/ charmonizer/src/Charmonizer/Probe/
charmonizer/src/Charmonizer/Test/ clownfish/ clownfish/li...
Added: incubator/lucy/trunk/import/core/KinoSearch/Store/FSFileHandle.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Store/FSFileHandle.c?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Store/FSFileHandle.c (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Store/FSFileHandle.c Thu Oct 14 08:48:04 2010
@@ -0,0 +1,619 @@
+#define C_KINO_FSFILEHANDLE
+#define C_KINO_FILEWINDOW
+#include "KinoSearch/Util/ToolSet.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <fcntl.h> // open, POSIX flags
+#include <stdarg.h>
+
+#ifdef CHY_HAS_UNISTD_H
+ #include <unistd.h> // close
+#endif
+
+#ifdef CHY_HAS_SYS_MMAN_H
+ #include <sys/mman.h>
+#elif defined(CHY_HAS_WINDOWS_H)
+ #include <windows.h>
+ #include <io.h>
+#else
+ #error "No support for memory mapped files"
+#endif
+
+#include "KinoSearch/Store/FSFileHandle.h"
+#include "KinoSearch/Store/FileWindow.h"
+
+// Convert FileHandle flags to POSIX flags.
+static INLINE int
+SI_posix_flags(uint32_t fh_flags)
+{
+ int posix_flags = 0;
+ if (fh_flags & FH_WRITE_ONLY) { posix_flags |= O_WRONLY; }
+ if (fh_flags & FH_READ_ONLY) { posix_flags |= O_RDONLY; }
+ if (fh_flags & FH_CREATE) { posix_flags |= O_CREAT; }
+ if (fh_flags & FH_EXCLUSIVE) { posix_flags |= O_EXCL; }
+#ifdef O_LARGEFILE
+ posix_flags |= O_LARGEFILE;
+#endif
+#ifdef _O_BINARY
+ posix_flags |= _O_BINARY;
+#endif
+ return posix_flags;
+}
+
+#define IS_64_BIT (SIZEOF_PTR == 8 ? true : false)
+
+// Memory map a region of the file with shared (read-only) permissions. If
+// the requested length is 0, return NULL. If an error occurs, return NULL
+// and set Err_error.
+static INLINE void*
+SI_map(FSFileHandle *self, int64_t offset, int64_t len);
+
+// Release a memory mapped region assigned by SI_map.
+static INLINE bool_t
+SI_unmap(FSFileHandle *self, char *ptr, int64_t len);
+
+// 32-bit or 64-bit inlined helpers for FSFH_window.
+static INLINE bool_t
+SI_window(FSFileHandle *self, FileWindow *window, int64_t offset, int64_t len);
+
+// Architecture- and OS- specific initialization for a read-only FSFileHandle.
+static INLINE bool_t
+SI_init_read_only(FSFileHandle *self);
+
+// Windows-specific routine needed for closing read-only handles.
+#ifdef CHY_HAS_WINDOWS_H
+static INLINE bool_t
+SI_close_win_handles(FSFileHandle *self);
+#endif
+
+FSFileHandle*
+FSFH_open(const CharBuf *path, uint32_t flags)
+{
+ FSFileHandle *self = (FSFileHandle*)VTable_Make_Obj(FSFILEHANDLE);
+ return FSFH_do_open(self, path, flags);
+}
+
+FSFileHandle*
+FSFH_do_open(FSFileHandle *self, const CharBuf *path, uint32_t flags)
+{
+ FH_do_open((FileHandle*)self, path, flags);
+ if (!path || !CB_Get_Size(path)) {
+ Err_set_error(Err_new(CB_newf("Missing required param 'path'")));
+ DECREF(self);
+ return NULL;
+ }
+
+ // Attempt to open file.
+ if (flags & FH_WRITE_ONLY) {
+ self->fd = open((char*)CB_Get_Ptr8(path), SI_posix_flags(flags), 0666);
+ if (self->fd == -1) {
+ self->fd = 0;
+ Err_set_error(Err_new(CB_newf("Attempt to open '%o' failed: %s",
+ path, strerror(errno))));
+ DECREF(self);
+ return NULL;
+ }
+ if (flags & FH_EXCLUSIVE) {
+ self->len = 0;
+ }
+ else {
+ // Derive length.
+ self->len = lseek64(self->fd, I64_C(0), SEEK_END);
+ if (self->len == -1) {
+ Err_set_error(Err_new(CB_newf("lseek64 on %o failed: %s",
+ self->path, strerror(errno))));
+ DECREF(self);
+ return NULL;
+ }
+ else {
+ int64_t check_val = lseek64(self->fd, I64_C(0), SEEK_SET);
+ if (check_val == -1) {
+ Err_set_error(Err_new(CB_newf("lseek64 on %o failed: %s",
+ self->path, strerror(errno))));
+ DECREF(self);
+ return NULL;
+ }
+ }
+ }
+ }
+ else if (flags & FH_READ_ONLY) {
+ if (SI_init_read_only(self)) {
+ // On 64-bit systems, map the whole file up-front.
+ if (IS_64_BIT && self->len) {
+ self->buf = (char*)SI_map(self, 0, self->len);
+ if (!self->buf) {
+ // An error occurred during SI_map, which has set
+ // Err_error for us already.
+ DECREF(self);
+ return NULL;
+ }
+ }
+ }
+ else {
+ DECREF(self);
+ return NULL;
+ }
+ }
+ else {
+ Err_set_error(Err_new(CB_newf(
+ "Must specify FH_READ_ONLY or FH_WRITE_ONLY to open '%o'",
+ path)));
+ DECREF(self);
+ return NULL;
+ }
+
+ return self;
+}
+
+bool_t
+FSFH_close(FSFileHandle *self)
+{
+ // On 64-bit systems, cancel the whole-file mapping.
+ if (IS_64_BIT && (self->flags & FH_READ_ONLY)) {
+ if ( !SI_unmap(self, self->buf, self->len) ) { return false; }
+ self->buf = NULL;
+ }
+
+ // Close system-specific handles.
+ if (self->fd) {
+ if (close(self->fd)) {
+ Err_set_error(Err_new(CB_newf("Failed to close file: %s",
+ strerror(errno))));
+ return false;
+ }
+ self->fd = 0;
+ }
+ #ifdef CHY_HAS_WINDOWS_H
+ if (self->win_fhandle) {
+ if (!SI_close_win_handles(self)) { return false; }
+ }
+ #endif
+
+ return true;
+}
+
+bool_t
+FSFH_write(FSFileHandle *self, const void *data, size_t len)
+{
+ if (len) {
+ // Write data, track file length, check for errors.
+ int64_t check_val = write(self->fd, data, len);
+ self->len += check_val;
+ if (check_val != len) {
+ if (check_val == -1) {
+ Err_set_error(Err_new(CB_newf(
+ "Error when writing %u64 bytes: %s",
+ (uint64_t)len, strerror(errno))));
+ }
+ else {
+ Err_set_error(Err_new(CB_newf(
+ "Attempted to write %u64 bytes, but wrote %i64",
+ (uint64_t)len, check_val)));
+ }
+ return false;
+ }
+ }
+
+ return true;
+}
+
+int64_t
+FSFH_length(FSFileHandle *self)
+{
+ return self->len;
+}
+
+bool_t
+FSFH_window(FSFileHandle *self, FileWindow *window, int64_t offset, int64_t len)
+{
+ const int64_t end = offset + len;
+ if (!(self->flags & FH_READ_ONLY)) {
+ Err_set_error(Err_new(CB_newf("Can't read from write-only handle")));
+ return false;
+ }
+ else if (offset < 0) {
+ Err_set_error(Err_new(CB_newf("Can't read from negative offset %i64",
+ offset)));
+ return false;
+ }
+ else if (end > self->len) {
+ Err_set_error(Err_new(CB_newf(
+ "Tried to read past EOF: offset %i64 + request %i64 > len %i64",
+ offset, len, self->len)));
+ return false;
+ }
+ else {
+ return SI_window(self, window, offset, len);
+ }
+}
+
+/********************************* 64-bit *********************************/
+
+#if IS_64_BIT
+
+static INLINE bool_t
+SI_window(FSFileHandle *self, FileWindow *window, int64_t offset, int64_t len)
+{
+ FileWindow_Set_Window(window, self->buf + offset, offset, len);
+ return true;
+}
+
+bool_t
+FSFH_release_window(FSFileHandle *self, FileWindow *window)
+{
+ UNUSED_VAR(self);
+ FileWindow_Set_Window(window, NULL, 0, 0);
+ return true;
+}
+
+bool_t
+FSFH_read(FSFileHandle *self, char *dest, int64_t offset, size_t len)
+{
+ const int64_t end = offset + len;
+
+ if (self->flags & FH_WRITE_ONLY) {
+ Err_set_error(Err_new(CB_newf(
+ "Can't read from write-only filehandle")));
+ return false;
+ }
+ if (offset < 0) {
+ Err_set_error(Err_new(CB_newf(
+ "Can't read from an offset less than 0 (%i64)", offset)));
+ return false;
+ }
+ else if (end > self->len) {
+ Err_set_error(Err_new(CB_newf(
+ "Tried to read past EOF: offset %i64 + request %u64 > len %i64",
+ offset, (uint64_t)len, self->len)));
+ return false;
+ }
+ memcpy(dest, self->buf + offset, len);
+ return true;
+}
+
+/********************************* 32-bit *********************************/
+
+#else
+
+static INLINE bool_t
+SI_window(FSFileHandle *self, FileWindow *window, int64_t offset, int64_t len)
+{
+ // Release the previously mmap'd region, if any.
+ FSFH_release_window(self, window);
+
+ {
+ // Start map on a page boundary. Ensure that the window is at
+ // least wide enough to view all the data spec'd in the original
+ // request.
+ const int64_t remainder = offset % self->page_size;
+ const int64_t adjusted_offset = offset - remainder;
+ const int64_t adjusted_len = len + remainder;
+ char *const buf
+ = (char*)SI_map(self, adjusted_offset, adjusted_len);
+ if (len && buf == NULL) {
+ return false;
+ }
+ else {
+ FileWindow_Set_Window(window, buf, adjusted_offset,
+ adjusted_len);
+ }
+ }
+
+ return true;
+}
+
+bool_t
+FSFH_release_window(FSFileHandle *self, FileWindow *window)
+{
+ if ( !SI_unmap(self, window->buf, window->len) ) { return false; }
+ FileWindow_Set_Window(window, NULL, 0, 0);
+ return true;
+}
+
+#endif // IS_64_BIT vs. 32-bit
+
+/********************************* UNIXEN *********************************/
+
+#ifdef CHY_HAS_SYS_MMAN_H
+
+static INLINE bool_t
+SI_init_read_only(FSFileHandle *self)
+{
+ // Open.
+ self->fd = open((char*)CB_Get_Ptr8(self->path),
+ SI_posix_flags(self->flags), 0666);
+ if (self->fd == -1) {
+ self->fd = 0;
+ Err_set_error(Err_new(CB_newf("Can't open '%o': %s", self->path,
+ strerror(errno))));
+ return false;
+ }
+
+ // Derive len.
+ self->len = lseek64(self->fd, I64_C(0), SEEK_END);
+ if (self->len == -1) {
+ Err_set_error(Err_new(CB_newf("lseek64 on %o failed: %s", self->path,
+ strerror(errno))));
+ return false;
+ }
+ else {
+ int64_t check_val = lseek64(self->fd, I64_C(0), SEEK_SET);
+ if (check_val == -1) {
+ Err_set_error(Err_new(CB_newf("lseek64 on %o failed: %s",
+ self->path, strerror(errno))));
+ return false;
+ }
+ }
+
+ // Get system page size.
+#if defined(_SC_PAGESIZE)
+ self->page_size = sysconf(_SC_PAGESIZE);
+#elif defined(_SC_PAGE_SIZE)
+ self->page_size = sysconf(_SC_PAGE_SIZE);
+#else
+ #error "Can't determine system memory page size"
+#endif
+
+ return true;
+}
+
+static INLINE void*
+SI_map(FSFileHandle *self, int64_t offset, int64_t len)
+{
+ void *buf = NULL;
+
+ if (len) {
+ // Read-only memory mapping.
+ buf = mmap(NULL, len, PROT_READ, MAP_SHARED, self->fd, offset);
+ if (buf == (void*)-1) {
+ Err_set_error(Err_new(CB_newf(
+ "mmap of offset %i64 and length %i64 (page size %i64) "
+ "against '%o' failed: %s", offset, len, self->page_size,
+ self->path, strerror(errno))));
+ return NULL;
+ }
+ }
+
+ return buf;
+}
+
+static INLINE bool_t
+SI_unmap(FSFileHandle *self, char *buf, int64_t len)
+{
+ if (buf != NULL) {
+ if (munmap(buf, len)) {
+ Err_set_error(Err_new(CB_newf("Failed to munmap '%o': %s",
+ self->path, strerror(errno))));
+ return false;
+ }
+ }
+ return true;
+}
+
+#if !IS_64_BIT
+bool_t
+FSFH_read(FSFileHandle *self, char *dest, int64_t offset, size_t len)
+{
+ int64_t check_val;
+
+ // Sanity check.
+ if (offset < 0) {
+ Err_set_error(Err_new(CB_newf(
+ "Can't read from an offset less than 0 (%i64)", offset)));
+ return false;
+ }
+
+ // Read.
+ check_val = pread64(self->fd, dest, len, offset);
+ if (check_val != (int64_t)len) {
+ if (check_val == -1) {
+ Err_set_error(Err_new(CB_newf(
+ "Tried to read %u64 bytes, got %i64: %s",
+ (uint64_t)len, check_val, strerror(errno))));
+ }
+ else {
+ Err_set_error(Err_new(CB_newf(
+ "Tried to read %u64 bytes, got %i64",
+ (uint64_t)len, check_val)));
+ }
+ return false;
+ }
+
+ return true;
+}
+#endif // !IS_64_BIT
+
+/********************************* WINDOWS **********************************/
+
+#elif defined(CHY_HAS_WINDOWS_H)
+
+static INLINE bool_t
+SI_init_read_only(FSFileHandle *self)
+{
+ LARGE_INTEGER large_int;
+ char *filepath = (char*)CB_Get_Ptr8(self->path);
+ SYSTEM_INFO sys_info;
+
+ // Get system page size.
+ GetSystemInfo(&sys_info);
+ self->page_size = sys_info.dwAllocationGranularity;
+
+ // Open.
+ self->win_fhandle = CreateFile(
+ filepath,
+ GENERIC_READ,
+ FILE_SHARE_READ,
+ NULL,
+ OPEN_EXISTING,
+ FILE_ATTRIBUTE_READONLY | FILE_FLAG_OVERLAPPED,
+ NULL
+ );
+ if (self->win_fhandle == INVALID_HANDLE_VALUE) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf("CreateFile for %o failed: %s",
+ self->path, win_error)));
+ FREEMEM(win_error);
+ return false;
+ }
+
+ // Derive len.
+ GetFileSizeEx(self->win_fhandle, &large_int);
+ self->len = large_int.QuadPart;
+ if (self->len < 0) {
+ Err_set_error(Err_new(CB_newf(
+ "GetFileSizeEx for %o returned a negative length: '%i64'",
+ self->path, self->len)));
+ return false;
+ }
+
+ // Init mapping handle.
+ self->buf = NULL;
+ if (self->len) {
+ self->win_maphandle = CreateFileMapping(self->win_fhandle, NULL,
+ PAGE_READONLY, 0, 0, NULL);
+ if (self->win_maphandle == NULL) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf(
+ "CreateFileMapping for %o failed: %s",
+ self->path, win_error)));
+ FREEMEM(win_error);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static INLINE void*
+SI_map(FSFileHandle *self, int64_t offset, int64_t len)
+{
+ void *buf = NULL;
+
+ if (len) {
+ // Read-only memory map.
+ uint64_t offs = (uint64_t)offset;
+ DWORD file_offset_hi = offs >> 32;
+ DWORD file_offset_lo = offs & 0xFFFFFFFF;
+ size_t amount = (size_t)len;
+ buf = MapViewOfFile(
+ self->win_maphandle,
+ FILE_MAP_READ,
+ file_offset_hi,
+ file_offset_lo,
+ amount
+ );
+ if (buf == NULL) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf(
+ "MapViewOfFile for %o failed: %s",
+ self->path, win_error)));
+ FREEMEM(win_error);
+ }
+ }
+
+ return buf;
+}
+
+static INLINE bool_t
+SI_unmap(FSFileHandle *self, char *buf, int64_t len)
+{
+ if (buf != NULL) {
+ if (!UnmapViewOfFile(buf)) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf("Failed to unmap '%o': %s",
+ self->path, win_error)));
+ FREEMEM(win_error);
+ return false;
+ }
+ }
+ return true;
+}
+
+static INLINE bool_t
+SI_close_win_handles(FSFileHandle *self)
+{
+ // Close both standard handle and mapping handle.
+ if (self->win_maphandle) {
+ if (!CloseHandle(self->win_maphandle)) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf(
+ "Failed to close file mapping handle: %s", win_error)));
+ FREEMEM(win_error);
+ return false;
+ }
+ self->win_maphandle = NULL;
+ }
+ if (self->win_fhandle) {
+ if (!CloseHandle(self->win_fhandle)) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf("Failed to close file handle: %s",
+ win_error)));
+ FREEMEM(win_error);
+ return false;
+ }
+ self->win_fhandle = NULL;
+ }
+
+ return true;
+}
+
+#if !IS_64_BIT
+bool_t
+FSFH_read(FSFileHandle *self, char *dest, int64_t offset, size_t len)
+{
+ BOOL check_val;
+ DWORD got;
+ OVERLAPPED read_op_state;
+ uint64_t offs = (uint64_t)offset;
+
+ read_op_state.hEvent = NULL;
+ read_op_state.OffsetHigh = offs >> 32;
+ read_op_state.Offset = offs & 0xFFFFFFFF;
+
+ // Sanity check.
+ if (offset < 0) {
+ Err_set_error(Err_new(CB_newf(
+ "Can't read from an offset less than 0 (%i64)", offset)));
+ return false;
+ }
+
+ // ReadFile() takes a DWORD (unsigned 32-bit integer) as a length
+ // argument, so throw a sensible error rather than wrap around.
+ if (len > U32_MAX) {
+ Err_set_error(Err_new(CB_newf(
+ "Can't read more than 4 GB (%u64)", (uint64_t)len)));
+ return false;
+ }
+
+ // Read.
+ check_val = ReadFile(self->win_fhandle, dest, len, &got, &read_op_state);
+ if (!check_val && GetLastError() == ERROR_IO_PENDING) {
+ // Read has been queued by the OS and will soon complete. Wait for
+ // it, since this is a blocking IO call from the point of the rest of
+ // the library.
+ check_val = GetOverlappedResult(self->win_fhandle, &read_op_state,
+ &got, TRUE);
+ }
+
+ // Verify that the read has succeeded by now.
+ if (!check_val) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf("Failed to read %u64 bytes: %s",
+ (uint64_t)len, win_error)));
+ FREEMEM(win_error);
+ return false;
+ }
+
+ return true;
+}
+#endif // !IS_64_BIT
+
+#endif // CHY_HAS_SYS_MMAN_H vs. CHY_HAS_WINDOWS_H
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Propchange: incubator/lucy/trunk/import/core/KinoSearch/Store/FSFileHandle.c
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/lucy/trunk/import/core/KinoSearch/Store/FSFileHandle.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Store/FSFileHandle.cfh?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Store/FSFileHandle.cfh (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Store/FSFileHandle.cfh Thu Oct 14 08:48:04 2010
@@ -0,0 +1,51 @@
+parcel KinoSearch cnick Kino;
+
+/** File system FileHandle.
+ */
+class KinoSearch::Store::FSFileHandle cnick FSFH
+ inherits KinoSearch::Store::FileHandle {
+
+ int fd;
+ void *win_fhandle;
+ void *win_maphandle;
+ int64_t len;
+ int64_t page_size;
+ char *buf;
+
+ /** Return a new FSFileHandle, or set Err_error and return NULL if
+ * something goes wrong.
+ *
+ * @param path Filepath.
+ * @param flags FileHandle constructor flags.
+ */
+ inert incremented nullable FSFileHandle*
+ open(const CharBuf *path = NULL, uint32_t flags);
+
+ inert nullable FSFileHandle*
+ do_open(FSFileHandle *self, const CharBuf *path = NULL, uint32_t flags);
+
+ bool_t
+ Window(FSFileHandle *self, FileWindow *window, int64_t offset, int64_t len);
+
+ bool_t
+ Release_Window(FSFileHandle *self, FileWindow *window);
+
+ bool_t
+ Read(FSFileHandle *self, char *dest, int64_t offset, size_t len);
+
+ bool_t
+ Write(FSFileHandle *self, const void *data, size_t len);
+
+ int64_t
+ Length(FSFileHandle *self);
+
+ bool_t
+ Close(FSFileHandle *self);
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Added: incubator/lucy/trunk/import/core/KinoSearch/Store/FSFolder.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Store/FSFolder.c?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Store/FSFolder.c (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Store/FSFolder.c Thu Oct 14 08:48:04 2010
@@ -0,0 +1,320 @@
+#define C_KINO_FSFOLDER
+#include "KinoSearch/Util/ToolSet.h"
+
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <sys/stat.h>
+
+#ifdef CHY_HAS_SYS_TYPES_H
+ #include <sys/types.h>
+#endif
+
+// For rmdir, (hard) link.
+#ifdef CHY_HAS_UNISTD_H
+ #include <unistd.h>
+#endif
+
+// For CreateHardLink.
+#ifdef CHY_HAS_WINDOWS_H
+ #include <windows.h>
+#endif
+
+// For mkdir, rmdir.
+#ifdef CHY_HAS_DIRECT_H
+ #include <direct.h>
+#endif
+
+#include "KinoSearch/Store/FSFolder.h"
+#include "KinoSearch/Store/CompoundFileReader.h"
+#include "KinoSearch/Store/CompoundFileWriter.h"
+#include "KinoSearch/Store/FSDirHandle.h"
+#include "KinoSearch/Store/FSFileHandle.h"
+#include "KinoSearch/Store/InStream.h"
+#include "KinoSearch/Store/OutStream.h"
+#include "KinoSearch/Util/IndexFileNames.h"
+
+// Return a ZombieCharBuf (cast to CharBuf) containing a platform-specific
+// absolute filepath.
+static CharBuf*
+S_fullpath(FSFolder *self, const CharBuf *path, void *allocation,
+ size_t alloc_size);
+#define FULLPATH(self, _path) \
+ S_fullpath(self, _path, alloca( \
+ ZCB_size() + CB_Get_Size(self->path) + CB_Get_Size(_path) + 10), \
+ ZCB_size() + CB_Get_Size(self->path) + CB_Get_Size(_path) + 10)
+
+// Return true if the supplied path is a directory.
+static bool_t
+S_dir_ok(const CharBuf *path);
+
+// Create a directory, or set Err_error and return false.
+bool_t
+S_create_dir(const CharBuf *path);
+
+// Return true unless the supplied path contains a slash.
+bool_t
+S_is_local_entry(const CharBuf *path);
+
+FSFolder*
+FSFolder_new(const CharBuf *path)
+{
+ FSFolder *self = (FSFolder*)VTable_Make_Obj(FSFOLDER);
+ return FSFolder_init(self, path);
+}
+
+FSFolder*
+FSFolder_init(FSFolder *self, const CharBuf *path)
+{
+ CharBuf *abs_path = FSFolder_absolutify(path);
+ Folder_init((Folder*)self, abs_path);
+ DECREF(abs_path);
+ return self;
+}
+
+void
+FSFolder_initialize(FSFolder *self)
+{
+ if (!S_dir_ok(self->path)) {
+ if (!S_create_dir(self->path)) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ }
+}
+
+bool_t
+FSFolder_check(FSFolder *self)
+{
+ return S_dir_ok(self->path);
+}
+
+FileHandle*
+FSFolder_local_open_filehandle(FSFolder *self, const CharBuf *name, uint32_t flags)
+{
+ CharBuf *fullpath = FULLPATH(self, name);
+ FSFileHandle *fh = FSFH_open(fullpath, flags);
+ if (!fh) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ return (FileHandle*)fh;
+}
+
+bool_t
+FSFolder_local_mkdir(FSFolder *self, const CharBuf *name)
+{
+ CharBuf *dir = FULLPATH(self, name);
+ bool_t result = S_create_dir(dir);
+ if (!result) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ return result;
+}
+
+DirHandle*
+FSFolder_local_open_dir(FSFolder *self)
+{
+ DirHandle *dh = (DirHandle*)FSDH_open(self->path);
+ if (!dh) { ERR_ADD_FRAME(Err_get_error()); }
+ return dh;
+}
+
+bool_t
+FSFolder_local_exists(FSFolder *self, const CharBuf *name)
+{
+ if (Hash_Fetch(self->entries, (Obj*)name)) {
+ return true;
+ }
+ else if (!S_is_local_entry(name)) {
+ return false;
+ }
+ else {
+ struct stat stat_buf;
+ CharBuf *fullpath = FULLPATH(self, name);
+ bool_t retval = false;
+ if (stat((char*)CB_Get_Ptr8(fullpath), &stat_buf) != -1)
+ retval = true;
+ return retval;
+ }
+}
+
+bool_t
+FSFolder_local_is_directory(FSFolder *self, const CharBuf *name)
+{
+ // Check for a cached object, then fall back to a system call.
+ Obj *elem = Hash_Fetch(self->entries, (Obj*)name);
+ if (elem && Obj_Is_A(elem, FOLDER)) {
+ return true;
+ }
+ else {
+ CharBuf *fullpath = FULLPATH(self, name);
+ bool_t result = S_dir_ok(fullpath);
+ return result;
+ }
+}
+
+bool_t
+FSFolder_rename(FSFolder *self, const CharBuf* from, const CharBuf *to)
+{
+ CharBuf *from_path = FULLPATH(self, from);
+ CharBuf *to_path = FULLPATH(self, to);
+ bool_t retval = !rename((char*)CB_Get_Ptr8(from_path),
+ (char*)CB_Get_Ptr8(to_path));
+ if (!retval) {
+ Err_set_error(Err_new(CB_newf("rename from '%o' to '%o' failed: %s",
+ from_path, to_path, strerror(errno))));
+ }
+ return retval;
+}
+
+bool_t
+FSFolder_hard_link(FSFolder *self, const CharBuf *from,
+ const CharBuf *to)
+{
+ CharBuf *from_path = FULLPATH(self, from);
+ CharBuf *to_path = FULLPATH(self, to);
+ char *from8 = (char*)CB_Get_Ptr8(from_path);
+ char *to8 = (char*)CB_Get_Ptr8(to_path);
+#ifdef CHY_HAS_UNISTD_H
+ bool_t retval;
+ if (-1 == link(from8, to8)) {
+ retval = false;
+ Err_set_error(Err_new(CB_newf(
+ "hard link for new file '%o' from '%o' failed: %s",
+ to_path, from_path, strerror(errno))));
+ }
+ else {
+ retval = true;
+ }
+#elif defined(CHY_HAS_WINDOWS_H)
+ bool_t retval = !!CreateHardLink(to8, from8, NULL);
+ if (!retval) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf(
+ "CreateHardLink for new file '%o' from '%o' failed: %s",
+ to_path, from_path, win_error)));
+ FREEMEM(win_error);
+ }
+#endif
+ return retval;
+}
+
+bool_t
+FSFolder_local_delete(FSFolder *self, const CharBuf *name)
+{
+ CharBuf *fullpath = FULLPATH(self, name);
+ char *path_ptr = (char*)CB_Get_Ptr8(fullpath);
+#ifdef CHY_REMOVE_ZAPS_DIRS
+ bool_t result = !remove(path_ptr);
+#else
+ bool_t result = !rmdir(path_ptr) || !remove(path_ptr);
+#endif
+ DECREF(Hash_Delete(self->entries, (Obj*)name));
+ return result;
+}
+
+void
+FSFolder_close(FSFolder *self)
+{
+ Hash_Clear(self->entries);
+}
+
+Folder*
+FSFolder_local_find_folder(FSFolder *self, const CharBuf *name)
+{
+ Folder *subfolder = NULL;
+ if (!name || !CB_Get_Size(name)) {
+ // No entity can be identified by NULL or empty string.
+ return NULL;
+ }
+ else if (!S_is_local_entry(name)) {
+ return NULL;
+ }
+ else if (CB_Starts_With_Str(name, ".", 1)) {
+ // Don't allow access outside of the main dir.
+ return NULL;
+ }
+ else if (NULL != (subfolder = (Folder*)Hash_Fetch(self->entries, (Obj*)name))) {
+ if (Folder_Is_A(subfolder, FOLDER)) {
+ return subfolder;
+ }
+ else {
+ return NULL;
+ }
+ }
+
+ {
+ CharBuf *fullpath = FULLPATH(self, name);
+ if (S_dir_ok(fullpath)) {
+ subfolder = (Folder*)FSFolder_new(fullpath);
+ if (!subfolder) {
+ THROW(ERR, "Failed to open FSFolder at '%o'", fullpath);
+ }
+ // Try to open a CompoundFileReader. On failure, just use the
+ // existing folder.
+ CharBuf *cfmeta_file = (CharBuf*)ZCB_WRAP_STR("cfmeta.json", 11);
+ if (Folder_Local_Exists(subfolder, cfmeta_file)) {
+ CompoundFileReader *cf_reader = CFReader_open(subfolder);
+ if (cf_reader) {
+ DECREF(subfolder);
+ subfolder = (Folder*)cf_reader;
+ }
+ }
+ Hash_Store(self->entries, (Obj*)name, (Obj*)subfolder);
+ return subfolder;
+ }
+ }
+
+ return NULL;
+}
+
+static CharBuf*
+S_fullpath(FSFolder *self, const CharBuf *path, void *allocation,
+ size_t alloc_size)
+{
+ CharBuf *fullpath = (CharBuf*)ZCB_newf(allocation, alloc_size, "%o%s%o",
+ self->path, DIR_SEP, path);
+ if (DIR_SEP[0] != '/') {
+ CB_Swap_Chars(fullpath, '/', DIR_SEP[0]);
+ }
+ return fullpath;
+}
+
+static bool_t
+S_dir_ok(const CharBuf *path)
+{
+ struct stat stat_buf;
+ if (stat((char*)CB_Get_Ptr8(path), &stat_buf) != -1) {
+ if (stat_buf.st_mode & S_IFDIR) return true;
+ }
+ return false;
+}
+
+bool_t
+S_create_dir(const CharBuf *path)
+{
+ if (-1 == chy_makedir((char*)CB_Get_Ptr8(path), 0777)) {
+ Err_set_error(Err_new(CB_newf(
+ "Couldn't create directory '%o': %s", path, strerror(errno))));
+ return false;
+ }
+ return true;
+}
+
+bool_t
+S_is_local_entry(const CharBuf *path)
+{
+ ZombieCharBuf *scratch = ZCB_WRAP(path);
+ uint32_t code_point;
+ while (0 != (code_point = ZCB_Nip_One(scratch))) {
+ if (code_point == '/') { return false; }
+ }
+ return true;
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Propchange: incubator/lucy/trunk/import/core/KinoSearch/Store/FSFolder.c
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/lucy/trunk/import/core/KinoSearch/Store/FSFolder.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Store/FSFolder.cfh?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Store/FSFolder.cfh (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Store/FSFolder.cfh Thu Oct 14 08:48:04 2010
@@ -0,0 +1,76 @@
+parcel KinoSearch cnick Kino;
+
+/** File System implementation of Folder.
+ *
+ * Implementation of KinoSearch::Store::Folder using a single file system
+ * directory and multiple files.
+ */
+
+class KinoSearch::Store::FSFolder inherits KinoSearch::Store::Folder {
+
+ inert incremented FSFolder*
+ new(const CharBuf *path);
+
+ /**
+ * @param path Location of the index. If the specified directory does
+ * not exist already, it will NOT be created, in order to prevent
+ * misconfigured read applications from spawning bogus files -- so it may
+ * be necessary to create the directory yourself.
+ */
+ public inert FSFolder*
+ init(FSFolder *self, const CharBuf *path);
+
+ /** Attempt to create the directory specified by <code>path</code>.
+ */
+ public void
+ Initialize(FSFolder *self);
+
+ /** Verify that <code>path</code> is a directory. TODO: check
+ * permissions.
+ */
+ public bool_t
+ Check(FSFolder *self);
+
+ public void
+ Close(FSFolder *self);
+
+ incremented nullable FileHandle*
+ Local_Open_FileHandle(FSFolder *self, const CharBuf *name,
+ uint32_t flags);
+
+ incremented nullable DirHandle*
+ Local_Open_Dir(FSFolder *self);
+
+ bool_t
+ Local_MkDir(FSFolder *self, const CharBuf *name);
+
+ bool_t
+ Local_Exists(FSFolder *self, const CharBuf *name);
+
+ bool_t
+ Local_Is_Directory(FSFolder *self, const CharBuf *name);
+
+ nullable Folder*
+ Local_Find_Folder(FSFolder *self, const CharBuf *name);
+
+ bool_t
+ Local_Delete(FSFolder *self, const CharBuf *name);
+
+ public bool_t
+ Rename(FSFolder *self, const CharBuf* from, const CharBuf *to);
+
+ public bool_t
+ Hard_Link(FSFolder *self, const CharBuf *from, const CharBuf *to);
+
+ /** Transform a relative path into an abolute path.
+ */
+ inert incremented CharBuf*
+ absolutify(const CharBuf *path);
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Added: incubator/lucy/trunk/import/core/KinoSearch/Store/FileHandle.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Store/FileHandle.c?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Store/FileHandle.c (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Store/FileHandle.c Thu Oct 14 08:48:04 2010
@@ -0,0 +1,56 @@
+#include <stdarg.h>
+
+#define C_KINO_FILEHANDLE
+#include "KinoSearch/Util/ToolSet.h"
+
+#include "KinoSearch/Store/FileHandle.h"
+
+int32_t FH_object_count = 0;
+
+FileHandle*
+FH_do_open(FileHandle *self, const CharBuf *path, uint32_t flags)
+{
+ self->path = path ? CB_Clone(path) : CB_new(0);
+ self->flags = flags;
+
+ // Track number of live FileHandles released into the wild.
+ FH_object_count++;
+
+ ABSTRACT_CLASS_CHECK(self, FILEHANDLE);
+ return self;
+}
+
+void
+FH_destroy(FileHandle *self)
+{
+ FH_Close(self);
+ DECREF(self->path);
+ SUPER_DESTROY(self, FILEHANDLE);
+
+ // Decrement count of FileHandle objects in existence.
+ FH_object_count--;
+}
+
+bool_t
+FH_grow(FileHandle *self, int64_t length)
+{
+ UNUSED_VAR(self);
+ UNUSED_VAR(length);
+ return true;
+}
+
+void
+FH_set_path(FileHandle *self, const CharBuf *path)
+{
+ CB_Mimic(self->path, (Obj*)path);
+}
+
+CharBuf*
+FH_get_path(FileHandle *self) { return self->path; }
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Propchange: incubator/lucy/trunk/import/core/KinoSearch/Store/FileHandle.c
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/lucy/trunk/import/core/KinoSearch/Store/FileHandle.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Store/FileHandle.cfh?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Store/FileHandle.cfh (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Store/FileHandle.cfh Thu Oct 14 08:48:04 2010
@@ -0,0 +1,143 @@
+parcel KinoSearch cnick Kino;
+
+/** Abstract class for reading and writing to files.
+ *
+ * FileHandle abstracts low-level unbuffered read/write and memory mapping
+ * operations. Implementations might reference data in a local file system, a
+ * RAM "file", a custom-constructed compound file, etc.
+ *
+ * Desired behaviors for a FileHandle are expressed via flags supplied to the
+ * constructor:
+ *
+ * * FH_READ_ONLY - Read only.
+ * * FH_WRITE_ONLY - Write only.
+ * * FH_CREATE - Create the file if it does not yet exist.
+ * * FH_EXCLUSIVE - The attempt to open the file should fail if the file
+ * already exists.
+ */
+
+abstract class KinoSearch::Store::FileHandle cnick FH
+ inherits KinoSearch::Object::Obj {
+
+ CharBuf *path;
+ uint32_t flags;
+
+ /** Abstract constructor.
+ *
+ * @param path The path to the file.
+ * @param flags A 32-bit integer with bits set to indicate desired
+ * behaviors.
+ */
+ inert nullable FileHandle*
+ do_open(FileHandle *self, const CharBuf *path = NULL, uint32_t flags);
+
+ /** Ensure that the FileWindow's buffer provides access to file data for
+ * <code>len</code> bytes starting at <code>offset</code>.
+ *
+ * @param window A FileWindow.
+ * @param offset File position to begin at.
+ * @param len Number of bytes to expose via the window.
+ * @return true on success, false on failure (sets Err_error)
+ */
+ abstract bool_t
+ Window(FileHandle *self, FileWindow *window, int64_t offset, int64_t len);
+
+ /** Clean up the FileWindow, doing whatever is necessary to free its
+ * buffer and reset its internal variables.
+ *
+ * @return true on success, false on failure (sets Err_error)
+ */
+ abstract bool_t
+ Release_Window(FileHandle *self, FileWindow *window);
+
+ /** Copy file content into the supplied buffer.
+ *
+ * @param dest Supplied memory.
+ * @param offset File position to begin at.
+ * @param len Number of bytes to copy.
+ * @return true on success, false on failure (sets Err_error)
+ */
+ abstract bool_t
+ Read(FileHandle *self, char *dest, int64_t offset, size_t len);
+
+ /** Write supplied content.
+ *
+ * @param data Content to write.
+ * @param len Number of bytes to write.
+ * @return true on success, false on failure (sets Err_error)
+ */
+ abstract bool_t
+ Write(FileHandle *self, const void *data, size_t len);
+
+ /** Return the current length of the file in bytes, or set Err_error and
+ * return -1 on failure.
+ */
+ abstract int64_t
+ Length(FileHandle *self);
+
+ /** Advisory call alerting the FileHandle that it should prepare to occupy
+ * <code>len</code> bytes. The default implementation is a no-op.
+ *
+ * @return true on success, false on failure (sets Err_error).
+ */
+ bool_t
+ Grow(FileHandle *self, int64_t len);
+
+ /** Close the FileHandle, possibly releasing resources. Implementations
+ * should be be able to handle multiple invocations, returning success
+ * unless something unexpected happens.
+ *
+ * @return true on success, false on failure (sets Err_error)
+ */
+ abstract bool_t
+ Close(FileHandle *self);
+
+ /** Set the object's <code>path</code> attribute.
+ */
+ void
+ Set_Path(FileHandle *self, const CharBuf *path);
+
+ /** Return the object's <code>path</code> attribute.
+ */
+ nullable CharBuf*
+ Get_Path(FileHandle *self);
+
+ /** Invokes Close(), but ignores whether it succeeds or fails.
+ */
+ public void
+ Destroy(FileHandle *self);
+}
+
+__C__
+
+#define KINO_FH_READ_ONLY 0x1
+#define KINO_FH_WRITE_ONLY 0x2
+#define KINO_FH_CREATE 0x4
+#define KINO_FH_EXCLUSIVE 0x8
+
+/* Integer which is incremented each time a FileHandle is created and decremented
+ * when a FileHandle is destroyed. Since so many classes use FileHandle objects,
+ * they're the canary in the coal mine for detecting object-destruction memory
+ * leaks.
+ */
+extern int32_t kino_FH_object_count;
+
+// Default size for the memory buffer used by both InStream and OutStream.
+#define KINO_IO_STREAM_BUF_SIZE 1024
+
+#ifdef KINO_USE_SHORT_NAMES
+ #define FH_object_count kino_FH_object_count
+ #define IO_STREAM_BUF_SIZE KINO_IO_STREAM_BUF_SIZE
+ #define FH_READ_ONLY KINO_FH_READ_ONLY
+ #define FH_WRITE_ONLY KINO_FH_WRITE_ONLY
+ #define FH_CREATE KINO_FH_CREATE
+ #define FH_EXCLUSIVE KINO_FH_EXCLUSIVE
+#endif
+__END_C__
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Added: incubator/lucy/trunk/import/core/KinoSearch/Store/FileWindow.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Store/FileWindow.c?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Store/FileWindow.c (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Store/FileWindow.c Thu Oct 14 08:48:04 2010
@@ -0,0 +1,44 @@
+#define C_KINO_FILEWINDOW
+#include "KinoSearch/Util/ToolSet.h"
+
+#include "KinoSearch/Store/FileWindow.h"
+
+FileWindow*
+FileWindow_new()
+{
+ FileWindow *self = (FileWindow*)VTable_Make_Obj(FILEWINDOW);
+ return FileWindow_init(self);
+}
+
+FileWindow*
+FileWindow_init(FileWindow *self)
+{
+ return self;
+}
+
+void
+FileWindow_set_offset(FileWindow *self, int64_t offset)
+{
+ if (self->buf != NULL) {
+ if (offset != self->offset) {
+ THROW(ERR, "Can't set offset to %i64 instead of %i64 unless buf "
+ "is NULL", offset, self->offset);
+ }
+ }
+ self->offset = offset;
+}
+
+void
+FileWindow_set_window(FileWindow *self, char *buf, int64_t offset, int64_t len)
+{
+ self->buf = buf;
+ self->offset = offset;
+ self->len = len;
+}
+
+/* Copyright 2008-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Propchange: incubator/lucy/trunk/import/core/KinoSearch/Store/FileWindow.c
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/lucy/trunk/import/core/KinoSearch/Store/FileWindow.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Store/FileWindow.cfh?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Store/FileWindow.cfh (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Store/FileWindow.cfh Thu Oct 14 08:48:04 2010
@@ -0,0 +1,29 @@
+parcel KinoSearch cnick Kino;
+
+/** A portion of a file, viewed as an array of bytes.
+ */
+class KinoSearch::Store::FileWindow inherits KinoSearch::Object::Obj {
+
+ char *buf;
+ int64_t offset;
+ int64_t len;
+
+ inert FileWindow*
+ init(FileWindow *self);
+
+ inert incremented FileWindow*
+ new();
+
+ void
+ Set_Offset(FileWindow *self, int64_t offset);
+
+ void
+ Set_Window(FileWindow *self, char *buf, int64_t offset, int64_t len);
+}
+
+/* Copyright 2008-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Added: incubator/lucy/trunk/import/core/KinoSearch/Store/Folder.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Store/Folder.c?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Store/Folder.c (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Store/Folder.c Thu Oct 14 08:48:04 2010
@@ -0,0 +1,487 @@
+#define C_KINO_FOLDER
+#include "KinoSearch/Util/ToolSet.h"
+#include <ctype.h>
+#include <limits.h>
+
+#ifndef SIZE_MAX
+ #define SIZE_MAX ((size_t)-1)
+#endif
+
+#include "KinoSearch/Store/Folder.h"
+#include "KinoSearch/Store/CompoundFileReader.h"
+#include "KinoSearch/Store/CompoundFileWriter.h"
+#include "KinoSearch/Store/DirHandle.h"
+#include "KinoSearch/Store/FileHandle.h"
+#include "KinoSearch/Store/InStream.h"
+#include "KinoSearch/Store/OutStream.h"
+#include "KinoSearch/Util/IndexFileNames.h"
+
+Folder*
+Folder_init(Folder *self, const CharBuf *path)
+{
+ // Init.
+ self->entries = Hash_new(16);
+
+ // Copy.
+ if (path == NULL) {
+ self->path = CB_new_from_trusted_utf8("", 0);
+ }
+ else {
+ // Copy path, strip trailing slash or equivalent.
+ self->path = CB_Clone(path);
+ if (CB_Ends_With_Str(self->path, DIR_SEP, strlen(DIR_SEP))) {
+ CB_Chop(self->path, 1);
+ }
+ }
+
+ ABSTRACT_CLASS_CHECK(self, FOLDER);
+ return self;
+}
+
+void
+Folder_destroy(Folder *self)
+{
+ DECREF(self->path);
+ DECREF(self->entries);
+ SUPER_DESTROY(self, FOLDER);
+}
+
+InStream*
+Folder_open_in(Folder *self, const CharBuf *path)
+{
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ InStream *instream = NULL;
+
+ if (enclosing_folder) {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ instream = Folder_Local_Open_In(enclosing_folder, (CharBuf*)name);
+ if (!instream) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+ else {
+ Err_set_error(Err_new(CB_newf("Invalid path: '%o'", path)));
+ }
+
+ return instream;
+}
+
+/* This method exists as a hook for CompoundFileReader to override; it is
+ * necessary because calling CFReader_Local_Open_FileHandle() won't find
+ * virtual files. No other class should need to override it. */
+InStream*
+Folder_local_open_in(Folder *self, const CharBuf *name)
+{
+ FileHandle *fh = Folder_Local_Open_FileHandle(self, name, FH_READ_ONLY);
+ InStream *instream = NULL;
+ if (fh) {
+ instream = InStream_open((Obj*)fh);
+ DECREF(fh);
+ if (!instream) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+ else {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ return instream;
+}
+
+OutStream*
+Folder_open_out(Folder *self, const CharBuf *path)
+{
+ const uint32_t flags = FH_WRITE_ONLY | FH_CREATE | FH_EXCLUSIVE;
+ FileHandle *fh = Folder_Open_FileHandle(self, path, flags);
+ OutStream *outstream = NULL;
+ if (fh) {
+ outstream = OutStream_open((Obj*)fh);
+ DECREF(fh);
+ if (!outstream) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+ else {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ return outstream;
+}
+
+FileHandle*
+Folder_open_filehandle(Folder *self, const CharBuf *path, uint32_t flags)
+{
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ FileHandle *fh = NULL;
+
+ if (enclosing_folder) {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ fh = Folder_Local_Open_FileHandle(enclosing_folder,
+ (CharBuf*)name, flags);
+ if (!fh) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+ else {
+ Err_set_error(Err_new(CB_newf("Invalid path: '%o'", path)));
+ }
+
+ return fh;
+}
+
+bool_t
+Folder_delete(Folder *self, const CharBuf *path)
+{
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ if (enclosing_folder) {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ bool_t result = Folder_Local_Delete(enclosing_folder, (CharBuf*)name);
+ return result;
+ }
+ else {
+ return false;
+ }
+}
+
+bool_t
+Folder_delete_tree(Folder *self, const CharBuf *path)
+{
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+
+ // Don't allow Folder to delete itself.
+ if (!path || !CB_Get_Size(path)) { return false; }
+
+ if (enclosing_folder) {
+ ZombieCharBuf *local = IxFileNames_local_part(path, ZCB_BLANK());
+ if (Folder_Local_Is_Directory(enclosing_folder, (CharBuf*)local)) {
+ Folder *inner_folder
+ = Folder_Local_Find_Folder(enclosing_folder, (CharBuf*)local);
+ DirHandle *dh = Folder_Local_Open_Dir(inner_folder);
+ if (dh) {
+ VArray *files = VA_new(20);
+ VArray *dirs = VA_new(20);
+ CharBuf *entry = DH_Get_Entry(dh);
+ while (DH_Next(dh)) {
+ VA_Push(files, (Obj*)CB_Clone(entry));
+ if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) {
+ VA_Push(dirs, (Obj*)CB_Clone(entry));
+ }
+ }
+ for (uint32_t i = 0, max = VA_Get_Size(dirs); i < max; i++) {
+ CharBuf *name = (CharBuf*)VA_Fetch(files, i);
+ bool_t success = Folder_Delete_Tree(inner_folder, name);
+ if (!success && Folder_Local_Exists(inner_folder, name)) {
+ break;
+ }
+ }
+ for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
+ CharBuf *name = (CharBuf*)VA_Fetch(files, i);
+ bool_t success = Folder_Local_Delete(inner_folder, name);
+ if (!success && Folder_Local_Exists(inner_folder, name)) {
+ break;
+ }
+ }
+ DECREF(dirs);
+ DECREF(files);
+ DECREF(dh);
+ }
+ }
+ return Folder_Local_Delete(enclosing_folder, (CharBuf*)local);
+ }
+ else {
+ // Return failure if the entry wasn't there in the first place.
+ return false;
+ }
+}
+
+static bool_t
+S_is_updir(CharBuf *path)
+{
+ if (CB_Equals_Str(path, ".", 1) || CB_Equals_Str(path, "..", 2)) {
+ return true;
+ }
+ else {
+ return false;
+ }
+}
+
+static void
+S_add_to_file_list(Folder *self, VArray *list, CharBuf *dir, CharBuf *prefix)
+{
+ size_t orig_prefix_size = CB_Get_Size(prefix);
+ DirHandle *dh = Folder_Open_Dir(self, dir);
+ CharBuf *entry;
+
+ if (!dh) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+
+ entry = DH_Get_Entry(dh);
+ while (DH_Next(dh)) { // Updates entry
+ if (!S_is_updir(entry)) {
+ CharBuf *relpath = CB_newf("%o%o", prefix, entry);
+ if (VA_Get_Size(list) == VA_Get_Capacity(list)) {
+ VA_Grow(list, VA_Get_Size(list) * 2);
+ }
+ VA_Push(list, (Obj*)relpath);
+
+ if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) {
+ CharBuf *subdir = CB_Get_Size(dir)
+ ? CB_newf("%o/%o", dir, entry)
+ : CB_Clone(entry);
+ CB_catf(prefix, "%o/", entry);
+ S_add_to_file_list(self, list, subdir, prefix); // recurse
+ CB_Set_Size(prefix, orig_prefix_size);
+ DECREF(subdir);
+ }
+ }
+ }
+
+ if (!DH_Close(dh)) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ DECREF(dh);
+}
+
+DirHandle*
+Folder_open_dir(Folder *self, const CharBuf *path)
+{
+ DirHandle *dh = NULL;
+ Folder *folder = Folder_Find_Folder(self,
+ path ? path : (CharBuf*)&EMPTY);
+ if (!folder) {
+ Err_set_error(Err_new(CB_newf("Invalid path: '%o'", path)));
+ }
+ else {
+ dh = Folder_Local_Open_Dir(folder);
+ if (!dh) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+ return dh;
+}
+
+bool_t
+Folder_mkdir(Folder *self, const CharBuf *path)
+{
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ bool_t result = false;
+
+ if (!CB_Get_Size(path)) {
+ Err_set_error(Err_new(CB_newf("Invalid path: '%o'", path)));
+ }
+ else if (!enclosing_folder) {
+ Err_set_error(Err_new(CB_newf(
+ "Can't recursively create dir %o", path)));
+ }
+ else {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ result = Folder_Local_MkDir(enclosing_folder, (CharBuf*)name);
+ if (!result) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+
+ return result;
+}
+
+bool_t
+Folder_exists(Folder *self, const CharBuf *path)
+{
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ bool_t retval = false;
+ if (enclosing_folder) {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ if (Folder_Local_Exists(enclosing_folder, (CharBuf*)name)) {
+ retval = true;
+ }
+ }
+ return retval;
+}
+
+bool_t
+Folder_is_directory(Folder *self, const CharBuf *path)
+{
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ bool_t retval = false;
+ if (enclosing_folder) {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ if (Folder_Local_Is_Directory(enclosing_folder, (CharBuf*)name)) {
+ retval = true;
+ }
+ }
+ return retval;
+}
+
+VArray*
+Folder_list(Folder *self, const CharBuf *path)
+{
+ Folder *local_folder = Folder_Find_Folder(self, path);
+ VArray *list = NULL;
+ DirHandle *dh = Folder_Local_Open_Dir(local_folder);
+ if (dh) {
+ CharBuf *entry = DH_Get_Entry(dh);
+ list = VA_new(32);
+ while (DH_Next(dh)) { VA_Push(list, (Obj*)CB_Clone(entry)); }
+ DECREF(dh);
+ }
+ else {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ return list;
+}
+
+VArray*
+Folder_list_r(Folder *self, const CharBuf *path)
+{
+ Folder *local_folder = Folder_Find_Folder(self, path);
+ VArray *list = VA_new(0);
+ if (local_folder) {
+ CharBuf *dir = CB_new(20);
+ CharBuf *prefix = CB_new(20);
+ if (path && CB_Get_Size(path)) {
+ CB_setf(prefix, "%o/", path);
+ }
+ S_add_to_file_list(local_folder, list, dir, prefix);
+ DECREF(prefix);
+ DECREF(dir);
+ }
+ return list;
+}
+
+ByteBuf*
+Folder_slurp_file(Folder *self, const CharBuf *path)
+{
+ InStream *instream = Folder_Open_In(self, path);
+ ByteBuf *retval = NULL;
+
+ if (!instream) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ else {
+ uint64_t length = InStream_Length(instream);
+
+ if (length >= SIZE_MAX) {
+ InStream_Close(instream);
+ DECREF(instream);
+ THROW(ERR, "File %o is too big to slurp (%u64 bytes)", path,
+ length);
+ }
+ else {
+ size_t size = (size_t)length;
+ char *ptr = (char*)MALLOCATE((size_t)size + 1);
+ InStream_Read_Bytes(instream, ptr, size);
+ ptr[size] = '\0';
+ retval = BB_new_steal_bytes(ptr, size, size + 1);
+ InStream_Close(instream);
+ DECREF(instream);
+ }
+ }
+
+ return retval;
+}
+
+CharBuf*
+Folder_get_path(Folder *self) { return self->path; }
+void
+Folder_set_path(Folder *self, const CharBuf *path)
+{
+ DECREF(self->path);
+ self->path = CB_Clone(path);
+}
+
+void
+Folder_consolidate(Folder *self, const CharBuf *path)
+{
+ Folder *folder = Folder_Find_Folder(self, path);
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ if (!folder) {
+ THROW(ERR, "Can't consolidate %o", path);
+ }
+ else if (Folder_Is_A(folder, COMPOUNDFILEREADER)) {
+ THROW(ERR, "Can't consolidate %o twice", path);
+ }
+ else {
+ CompoundFileWriter *cf_writer = CFWriter_new(folder);
+ CFWriter_Consolidate(cf_writer);
+ DECREF(cf_writer);
+ if (CB_Get_Size(path)) {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ CompoundFileReader *cf_reader = CFReader_open(folder);
+ if (!cf_reader) { RETHROW(INCREF(Err_get_error())); }
+ Hash_Store(enclosing_folder->entries, (Obj*)name,
+ (Obj*)cf_reader);
+ }
+ }
+}
+
+static Folder*
+S_enclosing_folder(Folder *self, ZombieCharBuf *path)
+{
+ size_t path_component_len = 0;
+ uint32_t code_point;
+
+ // Strip trailing slash.
+ if (ZCB_Code_Point_From(path, 0) == '/') { ZCB_Chop(path, 1); }
+
+ // Find first component of the file path.
+ ZombieCharBuf *scratch = ZCB_WRAP((CharBuf*)path);
+ ZombieCharBuf *path_component = ZCB_WRAP((CharBuf*)path);
+ while (0 != (code_point = ZCB_Nip_One(scratch))) {
+ if (code_point == '/') {
+ ZCB_Truncate(path_component, path_component_len);
+ ZCB_Nip(path, path_component_len + 1);
+ break;
+ }
+ path_component_len++;
+ }
+
+ // If we've eaten up the entire filepath, self is enclosing folder.
+ if (ZCB_Get_Size(scratch) == 0) { return self; }
+
+ {
+ Folder *local_folder
+ = Folder_Local_Find_Folder(self, (CharBuf*)path_component);
+ if (!local_folder) {
+ /* This element of the filepath doesn't exist, or it's not a
+ * directory. However, there are filepath characters left over,
+ * implying that this component ought to be a directory -- so the
+ * original file path is invalid. */
+ return NULL;
+ }
+
+ // This file path component is a folder. Recurse into it.
+ return S_enclosing_folder(local_folder, path);
+ }
+}
+
+Folder*
+Folder_enclosing_folder(Folder *self, const CharBuf *path)
+{
+ ZombieCharBuf *scratch = ZCB_WRAP(path);
+ return S_enclosing_folder(self, scratch);
+}
+
+Folder*
+Folder_find_folder(Folder *self, const CharBuf *path)
+{
+ if (!path || !CB_Get_Size(path)) {
+ return self;
+ }
+ else {
+ ZombieCharBuf *scratch = ZCB_WRAP(path);
+ Folder *enclosing_folder = S_enclosing_folder(self, scratch);
+ if (!enclosing_folder) {
+ return NULL;
+ }
+ else {
+ return Folder_Local_Find_Folder(enclosing_folder,
+ (CharBuf*)scratch);
+ }
+ }
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Propchange: incubator/lucy/trunk/import/core/KinoSearch/Store/Folder.c
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/lucy/trunk/import/core/KinoSearch/Store/Folder.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Store/Folder.cfh?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Store/Folder.cfh (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Store/Folder.cfh Thu Oct 14 08:48:04 2010
@@ -0,0 +1,262 @@
+parcel KinoSearch cnick Kino;
+
+/** Abstract class representing a directory.
+ *
+ * A "file" within a Folder might be a real file on disk -- or it might be a
+ * RAM buffer. Similarly, Delete() might delete a file from the file system, or
+ * a key-value pair from a hash, or something else.
+ *
+ * The archetypal implementation of Folder,
+ * L<FSFolder|KinoSearch::Store::FSFolder>, represents a directory on
+ * the file system holding a collection of files.
+ */
+abstract class KinoSearch::Store::Folder inherits KinoSearch::Object::Obj {
+
+ CharBuf *path;
+ Hash *entries;
+
+ public inert nullable Folder*
+ init(Folder *self, const CharBuf *path);
+
+ public void
+ Destroy(Folder *self);
+
+ /** Getter for <code>path</code> member var.
+ */
+ public CharBuf*
+ Get_Path(Folder *self);
+
+ /** Setter for <code>path</code> member var.
+ */
+ void
+ Set_Path(Folder *self, const CharBuf *path);
+
+ /** Open an OutStream, or set Err_error and return NULL on failure.
+ *
+ * @param path A relative filepath.
+ * @return an OutStream.
+ */
+ public incremented nullable OutStream*
+ Open_Out(Folder *self, const CharBuf *path);
+
+ /** Open an InStream, or set Err_error and return NULL on failure.
+ *
+ * @param path A relative filepath.
+ * @return an InStream.
+ */
+ public incremented nullable InStream*
+ Open_In(Folder *self, const CharBuf *path);
+
+ /** Open a FileHandle, or set Err_error and return NULL on failure.
+ *
+ * @param path A relative filepath.
+ * @param flags FileHandle flags.
+ * @return a FileHandle.
+ */
+ public incremented nullable FileHandle*
+ Open_FileHandle(Folder *self, const CharBuf *path, uint32_t flags);
+
+ /** Open a DirHandle or set Err_error and return NULL on failure.
+ *
+ * @param path Path to a subdirectory, relative to the Folder's path. If
+ * empty or NULL, returns a DirHandle for this Folder.
+ * @return a DirHandle.
+ */
+ public incremented nullable DirHandle*
+ Open_Dir(Folder *self, const CharBuf *path = NULL);
+
+ /** Create a subdirectory.
+ *
+ * @param path A relative filepath.
+ * @return true on success, false on failure (sets Err_error).
+ */
+ public bool_t
+ MkDir(Folder *self, const CharBuf *path);
+
+ /** List all local entries within a directory. Set Err_error and return
+ * NULL if something goes wrong.
+ *
+ * @param path A relative filepath optionally specifying a subdirectory.
+ * @return an unsorted array of filenames.
+ */
+ incremented VArray*
+ List(Folder *self, const CharBuf *path = NULL);
+
+ /** Recursively list all files and directories in the Folder.
+ *
+ * @param path A relative filepath optionally specifying a subdirectory.
+ * @return an unsorted array of relative filepaths.
+ */
+ incremented VArray*
+ List_R(Folder *self, const CharBuf *path = NULL);
+
+ /** Indicate whether an entity exists at <code>path</code>.
+ *
+ * @param path A relative filepath.
+ * @return true if <code>path</code> exists.
+ */
+ public bool_t
+ Exists(Folder *self, const CharBuf *path);
+
+ /** Indicate whether a directory exists at <code>path</code>.
+ *
+ * @param path A relative filepath.
+ * @return true if <code>path</code> is a directory.
+ */
+ bool_t
+ Is_Directory(Folder *self, const CharBuf *path);
+
+ /** Delete an entry from the folder.
+ *
+ * @param path A relative filepath.
+ * @return true if the deletion was successful.
+ */
+ public bool_t
+ Delete(Folder *self, const CharBuf *path);
+
+ /** Delete recursively, starting at <code>path</code>
+ *
+ * @param path A relative filepath specifying a file or subdirectory.
+ * @return true if the whole tree is deleted successfully, false if any
+ * part remains.
+ */
+ public bool_t
+ Delete_Tree(Folder *self, const CharBuf *path);
+
+ /** Rename a file or directory, or set Err_error and return false on
+ * failure. If an entry exists at <code>to</code>, the results are
+ * undefined.
+ *
+ * @param from The filepath prior to renaming.
+ * @param to The filepath after renaming.
+ * @return true on success, false on failure.
+ */
+ public abstract bool_t
+ Rename(Folder *self, const CharBuf *from, const CharBuf *to);
+
+ /** Create a hard link at path <code>to</code> pointing at the existing
+ * file <code>from</code>, or set Err_error and return false on failure.
+ *
+ * @return true on success, false on failure.
+ */
+ public abstract bool_t
+ Hard_Link(Folder *self, const CharBuf *from, const CharBuf *to);
+
+ /** Read a file and return its contents.
+ *
+ * @param path A relative filepath.
+ * @param return the file's contents.
+ */
+ public incremented ByteBuf*
+ Slurp_File(Folder *self, const CharBuf *path);
+
+ /** Collapse the contents of the directory into a compound file.
+ */
+ void
+ Consolidate(Folder *self, const CharBuf *path);
+
+ /** Given a filepath, return the Folder representing everything except
+ * the last component. E.g. the 'foo/bar' Folder for '/foo/bar/baz.txt',
+ * the 'foo' Folder for 'foo/bar', etc.
+ *
+ * If <code>path</code> is invalid, because an intermediate directory
+ * either doesn't exist or isn't a directory, return NULL.
+ */
+ nullable Folder*
+ Enclosing_Folder(Folder *self, const CharBuf *path);
+
+ /** Return the Folder at the subdirectory specified by <code>path</code>.
+ * If <code>path</code> is NULL or an empty string, return this Folder.
+ * If the entity at <code>path</code> either doesn't exist or isn't a
+ * subdirectory, return NULL.
+ *
+ * @param path A relative filepath specifying a subdirectory.
+ * @return A Folder.
+ */
+ nullable Folder*
+ Find_Folder(Folder *self, const CharBuf *path);
+
+ /** Perform implementation-specific initialization. For example: FSFolder
+ * creates its own directory.
+ */
+ public abstract void
+ Initialize(Folder *self);
+
+ /** Verify that operations may be performed on this Folder.
+ *
+ * @return true on success.
+ */
+ public abstract bool_t
+ Check(Folder *self);
+
+ /** Close the folder and release implementation-specific resources.
+ */
+ public abstract void
+ Close(Folder *self);
+
+ /** Open a FileHandle for a local file, or set Err_error and return NULL
+ * on failure.
+ */
+ abstract incremented nullable FileHandle*
+ Local_Open_FileHandle(Folder *self, const CharBuf *name, uint32_t flags);
+
+ /** Open an InStream for a local file, or set Err_error and return NULL on
+ * failure.
+ */
+ incremented nullable InStream*
+ Local_Open_In(Folder *self, const CharBuf *name);
+
+ /** Open a DirHandle to iterate over the local entries in this Folder, or
+ * set Err_error and return NULL on failure.
+ */
+ abstract incremented nullable DirHandle*
+ Local_Open_Dir(Folder *self);
+
+ /** Create a local subdirectory.
+ *
+ * @param name The name of the subdirectory.
+ * @return true on success, false on failure (sets Err_error)
+ */
+ abstract bool_t
+ Local_MkDir(Folder *self, const CharBuf *name);
+
+ /** Indicate whether a local entry exists for the supplied
+ * <code>name</code>.
+ *
+ * @param name The name of the local entry.
+ */
+ abstract bool_t
+ Local_Exists(Folder *self, const CharBuf *name);
+
+ /** Indicate whether a local subdirectory exists with the supplied
+ * <code>name</code>.
+ *
+ * @param name The name of the local subdirectory.
+ */
+ abstract bool_t
+ Local_Is_Directory(Folder *self, const CharBuf *name);
+
+ /** Return the Folder object representing the specified directory, if such
+ * a directory exists.
+ *
+ * @param name The name of a local directory.
+ * @return a Folder.
+ */
+ abstract nullable Folder*
+ Local_Find_Folder(Folder *self, const CharBuf *name);
+
+ /** Delete a local entry.
+ *
+ * @param name The name of the entry to be deleted.
+ * @return true if the deletion was successful.
+ */
+ abstract bool_t
+ Local_Delete(Folder *self, const CharBuf *name);
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Added: incubator/lucy/trunk/import/core/KinoSearch/Store/InStream.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Store/InStream.c?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Store/InStream.c (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Store/InStream.c Thu Oct 14 08:48:04 2010
@@ -0,0 +1,486 @@
+#define C_KINO_INSTREAM
+#define C_KINO_FILEWINDOW
+#include "KinoSearch/Util/ToolSet.h"
+
+#include "KinoSearch/Store/InStream.h"
+#include "KinoSearch/Store/FileHandle.h"
+#include "KinoSearch/Store/FSFileHandle.h"
+#include "KinoSearch/Store/FileWindow.h"
+#include "KinoSearch/Store/RAMFile.h"
+#include "KinoSearch/Store/RAMFileHandle.h"
+
+// Inlined version of InStream_Tell.
+static INLINE int64_t
+SI_tell(InStream *self);
+
+// Inlined version of InStream_Read_Bytes.
+static INLINE void
+SI_read_bytes(InStream *self, char* buf, size_t len);
+
+// Inlined version of InStream_Read_U8.
+static INLINE uint8_t
+SI_read_u8(InStream *self);
+
+// Ensure that the buffer contains exactly the specified amount of data.
+static void
+S_fill(InStream *self, int64_t amount);
+
+// Refill the buffer, with either IO_STREAM_BUF_SIZE bytes or all remaining
+// file content -- whichever is smaller. Throw an error if we're at EOF and
+// can't load at least one byte.
+static int64_t
+S_refill(InStream *self);
+
+InStream*
+InStream_open(Obj *file)
+{
+ InStream *self = (InStream*)VTable_Make_Obj(INSTREAM);
+ return InStream_do_open(self, file);
+}
+
+InStream*
+InStream_do_open(InStream *self, Obj *file)
+{
+ // Init.
+ self->buf = NULL;
+ self->limit = NULL;
+ self->offset = 0;
+ self->window = FileWindow_new();
+
+ // Obtain a FileHandle.
+ if (Obj_Is_A(file, FILEHANDLE)) {
+ self->file_handle = (FileHandle*)INCREF(file);
+ }
+ else if (Obj_Is_A(file, RAMFILE)) {
+ self->file_handle
+ = (FileHandle*)RAMFH_open(NULL, FH_READ_ONLY, (RAMFile*)file);
+ }
+ else if (Obj_Is_A(file, CHARBUF)) {
+ self->file_handle
+ = (FileHandle*)FSFH_open((CharBuf*)file, FH_READ_ONLY);
+ }
+ else {
+ Err_set_error(Err_new(CB_newf("Invalid type for param 'file': '%o'",
+ Obj_Get_Class_Name(file))));
+ DECREF(self);
+ return NULL;
+ }
+ if (!self->file_handle) {
+ ERR_ADD_FRAME(Err_get_error());
+ DECREF(self);
+ return NULL;
+ }
+
+ // Get length and filename from the FileHandle.
+ self->filename = CB_Clone(FH_Get_Path(self->file_handle));
+ self->len = FH_Length(self->file_handle);
+ if (self->len == -1) {
+ ERR_ADD_FRAME(Err_get_error());
+ DECREF(self);
+ return NULL;
+ }
+
+ return self;
+}
+
+void
+InStream_close(InStream *self)
+{
+ if (self->file_handle) {
+ FH_Release_Window(self->file_handle, self->window);
+ // Note that we don't close the FileHandle, because it's probably
+ // shared.
+ DECREF(self->file_handle);
+ self->file_handle = NULL;
+ }
+}
+
+void
+InStream_destroy(InStream *self)
+{
+ if (self->file_handle) {
+ InStream_Close(self);
+ }
+ DECREF(self->filename);
+ DECREF(self->window);
+ SUPER_DESTROY(self, INSTREAM);
+}
+
+InStream*
+InStream_reopen(InStream *self, const CharBuf *filename, int64_t offset,
+ int64_t len)
+{
+ if (!self->file_handle) {
+ THROW(ERR, "Can't Reopen() closed InStream %o", self->filename);
+ }
+ if (offset + len > FH_Length(self->file_handle)) {
+ THROW(ERR, "Offset + length too large (%i64 + %i64 > %i64)",
+ offset, len, FH_Length(self->file_handle));
+ }
+
+ InStream *evil_twin = (InStream*)VTable_Make_Obj(self->vtable);
+ InStream_do_open(evil_twin, (Obj*)self->file_handle);
+ if (filename != NULL) { CB_Mimic(evil_twin->filename, (Obj*)filename); }
+ evil_twin->offset = offset;
+ evil_twin->len = len;
+ InStream_Seek(evil_twin, 0);
+
+ return evil_twin;
+}
+
+InStream*
+InStream_clone(InStream *self)
+{
+ InStream *evil_twin = (InStream*)VTable_Make_Obj(self->vtable);
+ InStream_do_open(evil_twin, (Obj*)self->file_handle);
+ InStream_Seek(evil_twin, SI_tell(self));
+ return evil_twin;
+}
+
+CharBuf*
+InStream_get_filename(InStream *self) { return self->filename; }
+
+static int64_t
+S_refill(InStream *self)
+{
+ // Determine the amount to request.
+ const int64_t sub_file_pos = SI_tell(self);
+ const int64_t remaining = self->len - sub_file_pos;
+ const int64_t amount = remaining < IO_STREAM_BUF_SIZE
+ ? remaining
+ : IO_STREAM_BUF_SIZE;
+ if (!remaining) {
+ THROW(ERR, "Read past EOF of '%o' (offset: %i64 len: %i64)",
+ self->filename, self->offset, self->len);
+ }
+
+ // Make the request.
+ S_fill(self, amount);
+
+ return amount;
+}
+
+void
+InStream_refill(InStream *self)
+{
+ S_refill(self);
+}
+
+static void
+S_fill(InStream *self, int64_t amount)
+{
+ FileWindow *const window = self->window;
+ const int64_t virtual_file_pos = SI_tell(self);
+ const int64_t real_file_pos = virtual_file_pos + self->offset;
+ const int64_t remaining = self->len - virtual_file_pos;
+
+ // Throw an error if the requested amount would take us beyond EOF.
+ if (amount > remaining) {
+ THROW(ERR, "Read past EOF of %o (pos: %u64 len: %u64 request: %u64)",
+ self->filename, virtual_file_pos, self->len, amount);
+ }
+
+ // Make the request.
+ if (FH_Window(self->file_handle, window, real_file_pos, amount) ) {
+ char *const window_limit = window->buf + window->len;
+ self->buf = window->buf
+ - window->offset // theoretical start of real file
+ + self->offset // top of virtual file
+ + virtual_file_pos; // position within virtual file
+ self->limit = window_limit - self->buf > remaining
+ ? self->buf + remaining
+ : window_limit;
+ }
+ else {
+ Err *error = Err_get_error();
+ CB_catf(Err_Get_Mess(error), " (%o)", self->filename);
+ RETHROW(INCREF(error));
+ }
+}
+
+void
+InStream_fill(InStream *self, int64_t amount)
+{
+ S_fill(self, amount);
+}
+
+void
+InStream_seek(InStream *self, int64_t target)
+{
+ FileWindow *const window = self->window;
+ int64_t virtual_window_top = window->offset - self->offset;
+ int64_t virtual_window_end = virtual_window_top + window->len;
+
+ if (target < 0) {
+ THROW(ERR, "Can't Seek '%o' to negative target %i64", self->filename,
+ target);
+ }
+ // Seek within window if possible.
+ else if ( target >= virtual_window_top
+ && target <= virtual_window_end
+ ) {
+ self->buf = window->buf - window->offset + self->offset + target;
+ }
+ else if (target > self->len) {
+ THROW(ERR, "Can't Seek '%o' past EOF (%i64 > %i64)", self->filename,
+ target, self->len);
+ }
+ else {
+ // Target is outside window. Set all buffer and limit variables to
+ // NULL to trigger refill on the next read. Store the file position
+ // in the FileWindow's offset.
+ FH_Release_Window(self->file_handle, window);
+ self->buf = NULL;
+ self->limit = NULL;
+ FileWindow_Set_Offset(window, self->offset + target);
+ }
+}
+
+static INLINE int64_t
+SI_tell(InStream *self)
+{
+ FileWindow *const window = self->window;
+ int64_t pos_in_buf = PTR_TO_I64(self->buf) - PTR_TO_I64(window->buf);
+ return pos_in_buf + window->offset - self->offset;
+}
+
+int64_t
+InStream_tell(InStream *self)
+{
+ return SI_tell(self);
+}
+
+int64_t
+InStream_length(InStream *self)
+{
+ return self->len;
+}
+
+char*
+InStream_buf(InStream *self, size_t request)
+{
+ const int64_t bytes_in_buf = PTR_TO_I64(self->limit) - PTR_TO_I64(self->buf);
+
+ /* It's common for client code to overestimate how much is needed, because
+ * the request has to figure in worst-case for compressed data. However,
+ * if we can still serve them everything they request (e.g. they ask for 5
+ * bytes, they really need 1 byte, and there's 1k in the buffer), we can
+ * skip the following refill block. */
+ if ((int64_t)request > bytes_in_buf) {
+ const int64_t remaining_in_file = self->len - SI_tell(self);
+ int64_t amount = request;
+
+ // Try to bump up small requests.
+ if (amount < IO_STREAM_BUF_SIZE) { amount = IO_STREAM_BUF_SIZE; }
+
+ // Don't read past EOF.
+ if (remaining_in_file < amount) { amount = remaining_in_file; }
+
+ // Only fill if the recalculated, possibly smaller request exceeds the
+ // amount available in the buffer.
+ if (amount > bytes_in_buf) {
+ S_fill(self, amount);
+ }
+ }
+
+ return self->buf;
+}
+
+void
+InStream_advance_buf(InStream *self, char *buf)
+{
+ if (buf > self->limit) {
+ int64_t overrun = PTR_TO_I64(buf) - PTR_TO_I64(self->limit);
+ THROW(ERR, "Supplied value is %i64 bytes beyond end of buffer",
+ overrun);
+ }
+ else if (buf < self->buf) {
+ int64_t underrun = PTR_TO_I64(self->buf) - PTR_TO_I64(buf);
+ THROW(ERR, "Can't Advance_Buf backwards: (underrun: %i64))", underrun);
+ }
+ else {
+ self->buf = buf;
+ }
+}
+
+void
+InStream_read_bytes(InStream *self, char* buf, size_t len)
+{
+ SI_read_bytes(self, buf, len);
+}
+
+static INLINE void
+SI_read_bytes(InStream *self, char* buf, size_t len)
+{
+ const int64_t available = PTR_TO_I64(self->limit) - PTR_TO_I64(self->buf);
+ if (available >= (int64_t)len) {
+ // Request is entirely within buffer, so copy.
+ memcpy(buf, self->buf, len);
+ self->buf += len;
+ }
+ else {
+ // Pass along whatever we've got in the buffer.
+ if (available > 0) {
+ memcpy(buf, self->buf, (size_t)available);
+ buf += available;
+ len -= (size_t)available;
+ self->buf += available;
+ }
+
+ if (len < IO_STREAM_BUF_SIZE) {
+ // Ensure that we have enough mapped, then copy the rest.
+ int64_t got = S_refill(self);
+ if (got < len) {
+ int64_t orig_pos = SI_tell(self) - available;
+ int64_t orig_len = len + available;
+ THROW(ERR, "Read past EOF of %o (pos: %i64 len: %i64 "
+ "request: %i64)", self->filename, orig_pos,
+ self->len, orig_len);
+ }
+ memcpy(buf, self->buf, len);
+ self->buf += len;
+ }
+ else {
+ // Too big to handle via the buffer, so resort to a brute-force
+ // read.
+ const int64_t sub_file_pos = SI_tell(self);
+ const int64_t real_file_pos = sub_file_pos + self->offset;
+ bool_t success
+ = FH_Read(self->file_handle, buf, real_file_pos, len);
+ if (!success) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ InStream_seek(self, sub_file_pos + len);
+ }
+ }
+}
+
+int8_t
+InStream_read_i8(InStream *self)
+{
+ return (int8_t)SI_read_u8(self);
+}
+
+static INLINE uint8_t
+SI_read_u8(InStream *self)
+{
+ if (self->buf >= self->limit) { S_refill(self); }
+ return (uint8_t)*self->buf++;
+}
+
+uint8_t
+InStream_read_u8(InStream *self)
+{
+ return SI_read_u8(self);
+}
+
+static INLINE uint32_t
+SI_read_u32(InStream *self)
+{
+ uint32_t retval;
+ SI_read_bytes(self, (char*)&retval, 4);
+#ifdef LITTLE_END
+ retval = NumUtil_decode_bigend_u32((char*)&retval);
+#endif
+ return retval;
+}
+
+uint32_t
+InStream_read_u32(InStream *self)
+{
+ return SI_read_u32(self);
+}
+
+int32_t
+InStream_read_i32(InStream *self)
+{
+ return (int32_t)SI_read_u32(self);
+}
+
+static INLINE uint64_t
+SI_read_u64 (InStream *self)
+{
+ uint64_t retval;
+ SI_read_bytes(self, (char*)&retval, 8);
+#ifdef LITTLE_END
+ retval = NumUtil_decode_bigend_u64((char*)&retval);
+#endif
+ return retval;
+}
+
+uint64_t
+InStream_read_u64(InStream *self)
+{
+ return SI_read_u64(self);
+}
+
+int64_t
+InStream_read_i64(InStream *self)
+{
+ return (int64_t)SI_read_u64(self);
+}
+
+float
+InStream_read_f32(InStream *self)
+{
+ union { float f; uint32_t u32; } duo;
+ SI_read_bytes(self, (char*)&duo, sizeof(float));
+#ifdef LITTLE_END
+ duo.u32 = NumUtil_decode_bigend_u32(&duo.u32);
+#endif
+ return duo.f;
+}
+
+double
+InStream_read_f64(InStream *self)
+{
+ union { double d; uint64_t u64; } duo;
+ SI_read_bytes(self, (char*)&duo, sizeof(double));
+#ifdef LITTLE_END
+ duo.u64 = NumUtil_decode_bigend_u64(&duo.u64);
+#endif
+ return duo.d;
+}
+
+uint32_t
+InStream_read_c32(InStream *self)
+{
+ uint32_t retval = 0;
+ while (1) {
+ const uint8_t ubyte = SI_read_u8(self);
+ retval = (retval << 7) | (ubyte & 0x7f);
+ if ((ubyte & 0x80) == 0)
+ break;
+ }
+ return retval;
+}
+
+uint64_t
+InStream_read_c64(InStream *self)
+{
+ uint64_t retval = 0;
+ while (1) {
+ const uint8_t ubyte = SI_read_u8(self);
+ retval = (retval << 7) | (ubyte & 0x7f);
+ if ((ubyte & 0x80) == 0)
+ break;
+ }
+ return retval;
+}
+
+int
+InStream_read_raw_c64(InStream *self, char *buf)
+{
+ uint8_t *dest = (uint8_t*)buf;
+ do {
+ *dest = SI_read_u8(self);
+ } while ((*dest++ & 0x80) != 0);
+ return dest - (uint8_t*)buf;
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+
Propchange: incubator/lucy/trunk/import/core/KinoSearch/Store/InStream.c
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/lucy/trunk/import/core/KinoSearch/Store/InStream.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/core/KinoSearch/Store/InStream.cfh?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/core/KinoSearch/Store/InStream.cfh (added)
+++ incubator/lucy/trunk/import/core/KinoSearch/Store/InStream.cfh Thu Oct 14 08:48:04 2010
@@ -0,0 +1,186 @@
+parcel KinoSearch cnick Kino;
+
+/** Read index files.
+ *
+ * InStream objects are the primary interface for reading from index files.
+ * They are high-level (relatively speaking), media-agnostic wrappers
+ * around low-level, media-specific FileHandle objects.
+ *
+ * InStreams provide a number of routines for safely decoding common constructs
+ * such as big-endian or compressed integers; for the most part, these
+ * routines throw exceptions rather than require manual checking of return
+ * values for error conditions.
+ *
+ * Multiple InStream objects often share the same underlying FileHandle; this
+ * practice is safe because InStreams do not modify or rely upon the file
+ * position or other state within the FileHandle.
+ */
+class KinoSearch::Store::InStream inherits KinoSearch::Object::Obj {
+
+ int64_t offset;
+ int64_t len;
+ char *buf;
+ char *limit;
+ CharBuf *filename;
+ FileHandle *file_handle;
+ FileWindow *window;
+
+ inert incremented nullable InStream*
+ open(Obj *file);
+
+ /** Return a new InStream, or set Err_error and return NULL on failure.
+ *
+ * @param file A FileHandle, a file path, or a RAMFile.
+ */
+ inert nullable InStream*
+ do_open(InStream *self, Obj *file);
+
+ /** Clone the instream, but specify a new offset, length, and possibly
+ * filename. Initial file position will be set to the top of the file
+ * (taking <code>offset</code> into account).
+ *
+ * @param filename An alias filename. If NULL, the filename of the
+ * underlying FileHandle will be used.
+ * @param offset Top of the file as seen by the new InStream, in bytes
+ * from the top of the file as seen by the underlying FileHandle.
+ * @param len Length of the file as seen by the new InStream.
+ */
+ incremented InStream*
+ Reopen(InStream *self, const CharBuf *filename = NULL, int64_t offset,
+ int64_t len);
+
+ /** Clone the InStream. Clones share the same underlying FileHandle and
+ * start at the current file position, but are able to seek and read
+ * independently.
+ */
+ public incremented InStream*
+ Clone(InStream *self);
+
+ /** Decrement the number of streams using the underlying FileHandle. When
+ * the number drops to zero, possibly release system resources.
+ */
+ void
+ Close(InStream *self);
+
+ public void
+ Destroy(InStream *self);
+
+ /** Seek to <code>target</code>.
+ */
+ final void
+ Seek(InStream *self, int64_t target);
+
+ /** Return the current file position.
+ */
+ final int64_t
+ Tell(InStream *self);
+
+ /** Return the length of the "file" in bytes.
+ */
+ final int64_t
+ Length(InStream *self);
+
+ /** Fill the InStream's buffer, letting the FileHandle decide how many bytes
+ * of data to fill it with.
+ */
+ void
+ Refill(InStream *self);
+
+ /** Pour an exact number of bytes into the InStream's buffer.
+ */
+ void
+ Fill(InStream *self, int64_t amount);
+
+ /** Get the InStream's buffer. Check to see whether <code>request</code>
+ * bytes are already in the buffer. If not, fill the buffer with either
+ * <code>request</code> bytes or the number of bytes remaining before EOF,
+ * whichever is smaller.
+ *
+ * @param request Advisory byte size request.
+ * @return Pointer to the InStream's internal buffer.
+ */
+ final char*
+ Buf(InStream *self, size_t request);
+
+ /** Set the buf to a new value, checking for overrun. The idiom is for
+ * the caller to call Buf(), use no more bytes than requested, then use
+ * Advance_Buf() to update the InStream object.
+ */
+ final void
+ Advance_Buf(InStream *self, char *buf);
+
+ /** Read <code>len</code> bytes from the InStream into <code>buf</code>.
+ */
+ final void
+ Read_Bytes(InStream *self, char *buf, size_t len);
+
+ /** Read a signed 8-bit integer.
+ */
+ final int8_t
+ Read_I8(InStream *self);
+
+ /** Read an unsigned 8-bit integer.
+ */
+ final uint8_t
+ Read_U8(InStream *self);
+
+ /** Read a signed 32-bit integer.
+ */
+ final int32_t
+ Read_I32(InStream *self);
+
+ /** Read an unsigned 32-bit integer.
+ */
+ final uint32_t
+ Read_U32(InStream *self);
+
+ /** Read a signed 64-bit integer.
+ */
+ final int64_t
+ Read_I64(InStream *self);
+
+ /** Read an unsigned 64-bit integer.
+ */
+ final uint64_t
+ Read_U64(InStream *self);
+
+ /** Read an IEEE 764 32-bit floating point number.
+ */
+ final float
+ Read_F32(InStream *self);
+
+ /** Read an IEEE 764 64-bit floating point number.
+ */
+ final double
+ Read_F64(InStream *self);
+
+ /** Read in a compressed 32-bit unsigned integer.
+ */
+ uint32_t
+ Read_C32(InStream *self);
+
+ /** Read a 64-bit integer, using the same encoding as a C32 but occupying
+ * as many as 10 bytes.
+ */
+ final uint64_t
+ Read_C64(InStream *self);
+
+ /** Read the bytes for a C32/C64 into <code>buf</code>. Return the number
+ * of bytes read. The caller must ensure that sufficient space exists in
+ * <code>buf</code> (worst case is 10 bytes).
+ */
+ final int
+ Read_Raw_C64(InStream *self, char *buf);
+
+ /** Accessor for filename member.
+ */
+ CharBuf*
+ Get_Filename(InStream *self);
+}
+
+/* Copyright 2006-2010 Marvin Humphrey
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * under the same terms as Perl itself.
+ */
+