You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2010/12/03 15:54:27 UTC
svn commit: r1041844 - in /lucene/dev/trunk/lucene: contrib/
contrib/misc/src/java/org/apache/lucene/store/
src/test/org/apache/lucene/search/
Author: rmuir
Date: Fri Dec 3 14:54:27 2010
New Revision: 1041844
URL: http://svn.apache.org/viewvc?rev=1041844&view=rev
Log:
LUCENE-2791: add WindowsDirectory
Added:
lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.cpp (with props)
lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java (with props)
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSearchWithThreads.java (with props)
Modified:
lucene/dev/trunk/lucene/contrib/CHANGES.txt
Modified: lucene/dev/trunk/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/CHANGES.txt?rev=1041844&r1=1041843&r2=1041844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/contrib/CHANGES.txt Fri Dec 3 14:54:27 2010
@@ -32,6 +32,11 @@ New Features
* LUCENE-2507: Added DirectSpellChecker, which retrieves correction candidates directly
from the term dictionary using levenshtein automata. (Robert Muir)
+ * LUCENE-2791: Added WindowsDirectory, a Windows-specific Directory impl
+ that doesn't synchronize on the file handle. This can be useful to
+ avoid the performance problems of SimpleFSDirectory and NIOFSDirectory.
+ (Robert Muir, Simon Willnauer, Uwe Schindler, Michael McCandless)
+
API Changes
* LUCENE-2606: Changed RegexCapabilities interface to fix thread
Added: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.cpp
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.cpp?rev=1041844&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.cpp (added)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.cpp Fri Dec 3 14:54:27 2010
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+#include <jni.h>
+#include "windows.h"
+
+/**
+ * Windows Native IO methods.
+ */
+extern "C" {
+
+/**
+ * Utility to format a Windows system error code into an exception.
+ */
+void throwIOException(JNIEnv *env, DWORD error)
+{
+ jclass ioex;
+ char *msg;
+
+ ioex = env->FindClass("java/io/IOException");
+
+ if (ioex != NULL) {
+ FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+ NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR) &msg, 0, NULL );
+ env->ThrowNew(ioex, msg);
+ LocalFree(msg);
+ }
+}
+
+/**
+ * Utility to throw Exceptions on bad input
+ */
+void throwException(JNIEnv *env, const char *clazz, const char *msg)
+{
+ jclass exc = env->FindClass(clazz);
+
+ if (exc != NULL) {
+ env->ThrowNew(exc, msg);
+ }
+}
+
+/**
+ * Opens a handle to a file.
+ *
+ * Class: org_apache_lucene_store_WindowsDirectory
+ * Method: open
+ * Signature: (Ljava/lang/String;)J
+ */
+JNIEXPORT jlong JNICALL Java_org_apache_lucene_store_WindowsDirectory_open
+ (JNIEnv *env, jclass ignored, jstring filename)
+{
+ char *fname;
+ HANDLE handle;
+ jboolean isCopy;
+
+ if (filename == NULL) {
+ throwException(env, "java/lang/NullPointerException", "filename cannot be null");
+ return -1;
+ }
+
+ fname = (char *) env->GetStringUTFChars(filename, &isCopy);
+
+ if (fname == NULL) {
+ throwException(env, "java/lang/IllegalArgumentException", "invalid filename");
+ return -1;
+ }
+
+ handle = CreateFile(fname, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE,
+ NULL, OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS, NULL);
+
+ if (isCopy) {
+ env->ReleaseStringUTFChars(filename, fname);
+ }
+
+ if (handle == INVALID_HANDLE_VALUE) {
+ throwIOException(env, GetLastError());
+ return -1;
+ }
+
+ return (jlong) handle;
+}
+
+/**
+ * Reads data into the byte array, starting at offset, for length characters.
+ * The read is positioned at pos.
+ *
+ * Class: org_apache_lucene_store_WindowsDirectory
+ * Method: read
+ * Signature: (J[BIIJ)I
+ */
+JNIEXPORT jint JNICALL Java_org_apache_lucene_store_WindowsDirectory_read
+ (JNIEnv *env, jclass ignored, jlong fd, jbyteArray bytes, jint offset, jint length, jlong pos)
+{
+ OVERLAPPED io = { 0 };
+ DWORD numRead = -1;
+
+ io.Offset = (DWORD) (pos & 0xFFFFFFFF);
+ io.OffsetHigh = (DWORD) ((pos >> 0x20) & 0x7FFFFFFF);
+
+ if (bytes == NULL) {
+ throwException(env, "java/lang/NullPointerException", "bytes cannot be null");
+ return -1;
+ }
+
+ if (length <= 2048) { /* For small buffers, avoid GetByteArrayElements' copy */
+ char buffer[length];
+
+ if (ReadFile((HANDLE) fd, &buffer, length, &numRead, &io)) {
+ env->SetByteArrayRegion(bytes, offset, numRead, (const jbyte *) buffer);
+ } else {
+ throwIOException(env, GetLastError());
+ numRead = -1;
+ }
+
+ } else {
+ jboolean isCopy;
+ jbyte *buffer = env->GetByteArrayElements (bytes, &isCopy);
+
+ if (!ReadFile((HANDLE) fd, (void *)(buffer+offset), length, &numRead, &io)) {
+ throwIOException(env, GetLastError());
+ numRead = -1;
+ }
+
+ if (isCopy == JNI_TRUE) {
+ env->ReleaseByteArrayElements(bytes, buffer, numRead == 0 || numRead == -1 ? JNI_ABORT : 0);
+ }
+ }
+
+ return numRead;
+}
+
+/**
+ * Closes a handle to a file
+ *
+ * Class: org_apache_lucene_store_WindowsDirectory
+ * Method: close
+ * Signature: (J)V
+ */
+JNIEXPORT void JNICALL Java_org_apache_lucene_store_WindowsDirectory_close
+ (JNIEnv *env, jclass ignored, jlong fd)
+{
+ if (!CloseHandle((HANDLE) fd)) {
+ throwIOException(env, GetLastError());
+ }
+}
+
+/**
+ * Returns the length in bytes of a file.
+ *
+ * Class: org_apache_lucene_store_WindowsDirectory
+ * Method: length
+ * Signature: (J)J
+ */
+JNIEXPORT jlong JNICALL Java_org_apache_lucene_store_WindowsDirectory_length
+ (JNIEnv *env, jclass ignored, jlong fd)
+{
+ BY_HANDLE_FILE_INFORMATION info;
+
+ if (GetFileInformationByHandle((HANDLE) fd, (LPBY_HANDLE_FILE_INFORMATION) &info)) {
+ return (jlong) (((DWORDLONG) info.nFileSizeHigh << 0x20) + info.nFileSizeLow);
+ } else {
+ throwIOException(env, GetLastError());
+ return -1;
+ }
+}
+
+} /* extern "C" */
Added: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java?rev=1041844&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java (added)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java Fri Dec 3 14:54:27 2010
@@ -0,0 +1,123 @@
+package org.apache.lucene.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Native {@link Directory} implementation for Microsoft Windows.
+ * <p>
+ * Steps:
+ * <ol>
+ * <li>Compile the source code to create WindowsDirectory.dll:
+ * <blockquote>
+ * c:\mingw\bin\g++ -Wall -D_JNI_IMPLEMENTATION_ -Wl,--kill-at
+ * -I"%JAVA_HOME%\include" -I"%JAVA_HOME%\include\win32" -static-libgcc
+ * -static-libstdc++ -shared WindowsDirectory.cpp -o WindowsDirectory.dll
+ * </blockquote>
+ * For 64-bit JREs, use mingw64, with the -m64 option.
+ * <li>Put WindowsDirectory.dll into some directory in your windows PATH
+ * <li>Open indexes with WindowsDirectory and use it.
+ * </p>
+ * @lucene.experimental
+ */
+public class WindowsDirectory extends FSDirectory {
+
+ static {
+ System.loadLibrary("WindowsDirectory");
+ }
+
+ /** Create a new WindowsDirectory for the named location.
+ *
+ * @param path the path of the directory
+ * @param lockFactory the lock factory to use, or null for the default
+ * ({@link NativeFSLockFactory});
+ * @throws IOException
+ */
+ public WindowsDirectory(File path, LockFactory lockFactory) throws IOException {
+ super(path, lockFactory);
+ }
+
+ /** Create a new WindowsDirectory for the named location and {@link NativeFSLockFactory}.
+ *
+ * @param path the path of the directory
+ * @throws IOException
+ */
+ public WindowsDirectory(File path) throws IOException {
+ super(path, null);
+ }
+
+ public IndexInput openInput(String name, int bufferSize) throws IOException {
+ ensureOpen();
+ return new WindowsIndexInput(new File(getDirectory(), name), bufferSize);
+ }
+
+ protected static class WindowsIndexInput extends BufferedIndexInput {
+ private final long fd;
+ private final long length;
+ boolean isClone;
+ boolean isOpen;
+
+ public WindowsIndexInput(File file, int bufferSize) throws IOException {
+ super(bufferSize);
+ fd = WindowsDirectory.open(file.getPath());
+ length = WindowsDirectory.length(fd);
+ isOpen = true;
+ }
+
+ protected void readInternal(byte[] b, int offset, int length) throws IOException {
+ if (WindowsDirectory.read(fd, b, offset, length, getFilePointer()) != length)
+ throw new IOException("Read past EOF");
+ }
+
+ protected void seekInternal(long pos) throws IOException {
+ }
+
+ public synchronized void close() throws IOException {
+ // NOTE: we synchronize and track "isOpen" because Lucene sometimes closes IIs twice!
+ if (!isClone && isOpen) {
+ WindowsDirectory.close(fd);
+ isOpen = false;
+ }
+ }
+
+ public long length() {
+ return length;
+ }
+
+ @Override
+ public Object clone() {
+ WindowsIndexInput clone = (WindowsIndexInput)super.clone();
+ clone.isClone = true;
+ return clone;
+ }
+ }
+
+ /** Opens a handle to a file. */
+ private static native long open(String filename) throws IOException;
+
+ /** Reads data from a file at pos into bytes */
+ private static native int read(long fd, byte bytes[], int offset, int length, long pos) throws IOException;
+
+ /** Closes a handle to a file */
+ private static native void close(long fd) throws IOException;
+
+ /** Returns the length of a file */
+ private static native long length(long fd) throws IOException;
+}
Added: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSearchWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSearchWithThreads.java?rev=1041844&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSearchWithThreads.java (added)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSearchWithThreads.java Fri Dec 3 14:54:27 2010
@@ -0,0 +1,109 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestSearchWithThreads extends LuceneTestCase {
+
+ final int NUM_DOCS = 10000;
+ final int NUM_SEARCH_THREADS = 5;
+ final int RUN_TIME_MSEC = 1000 * RANDOM_MULTIPLIER;
+
+ public void test() throws Exception {
+ final Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random, dir);
+
+ final long startTime = System.currentTimeMillis();
+
+ // TODO: replace w/ the @nightly test data; make this
+ // into an optional @nightly stress test
+ final Document doc = new Document();
+ final Field body = newField("body", "", Field.Index.ANALYZED);
+ doc.add(body);
+ final StringBuilder sb = new StringBuilder();
+ for(int docCount=0;docCount<NUM_DOCS*RANDOM_MULTIPLIER;docCount++) {
+ final int numTerms = random.nextInt(10);
+ for(int termCount=0;termCount<numTerms;termCount++) {
+ sb.append(random.nextBoolean() ? "aaa" : "bbb");
+ sb.append(' ');
+ }
+ body.setValue(sb.toString());
+ w.addDocument(doc);
+ sb.delete(0, sb.length());
+ }
+ final IndexReader r = w.getReader();
+ w.close();
+
+ final long endTime = System.currentTimeMillis();
+ System.out.println("BUILD took " + (endTime-startTime));
+
+ final IndexSearcher s = new IndexSearcher(r);
+
+ final AtomicBoolean failed = new AtomicBoolean();
+ final long stopAt = System.currentTimeMillis() + RUN_TIME_MSEC;
+ final AtomicLong netSearch = new AtomicLong();
+
+ Thread[] threads = new Thread[NUM_SEARCH_THREADS];
+ for(int threadID=0;threadID<NUM_SEARCH_THREADS;threadID++) {
+ threads[threadID] = new Thread() {
+ TotalHitCountCollector col = new TotalHitCountCollector();
+ @Override
+ public void run() {
+ try {
+ long totHits = 0;
+ long totSearch = 0;
+ while(System.currentTimeMillis() < stopAt && !failed.get()) {
+ s.search(new TermQuery(new Term("body", "aaa")), col);
+ totHits += col.getTotalHits();
+ s.search(new TermQuery(new Term("body", "bbb")), col);
+ totHits += col.getTotalHits();
+ totSearch++;
+ }
+ assertTrue(totHits > 0);
+ netSearch.addAndGet(totSearch);
+ } catch (Exception exc) {
+ failed.set(true);
+ throw new RuntimeException(exc);
+ }
+ }
+ };
+ threads[threadID].setDaemon(true);
+ threads[threadID].start();
+ }
+
+ for(int threadID=0;threadID<NUM_SEARCH_THREADS;threadID++) {
+ threads[threadID].join();
+ }
+ System.out.println(NUM_SEARCH_THREADS + " threads did " + netSearch.get() + " searches");
+
+ s.close();
+ r.close();
+ dir.close();
+ }
+}