You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2020/11/16 08:40:14 UTC
[lucene-solr] branch master updated: LUCENE-8982: Separate out
native code to another module to allow cpp build with gradle (#2068)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new ebc87a8 LUCENE-8982: Separate out native code to another module to allow cpp build with gradle (#2068)
ebc87a8 is described below
commit ebc87a8a27f3b3bd89ea7c38c8b701d94e50788d
Author: zacharymorn <za...@yahoo.com>
AuthorDate: Mon Nov 16 00:40:03 2020 -0800
LUCENE-8982: Separate out native code to another module to allow cpp build with gradle (#2068)
* LUCENE-8982: Separate out native code to another module to allow cpp build with gradle
---
build.gradle | 3 +
gradle/native/disable-native.gradle | 82 ++++++++++++++++++++++
gradle/testing/randomization/policies/tests.policy | 4 ++
lucene/CHANGES.txt | 4 ++
lucene/misc/build.gradle | 4 +-
lucene/misc/native/build.gradle | 69 ++++++++++++++++++
.../src/main/posix}/NativePosixUtil.cpp | 20 +++---
.../src/main/windows}/WindowsDirectory.cpp | 16 ++---
.../apache/lucene/misc/store/NativePosixUtil.java | 2 +-
.../lucene/misc/store/NativeUnixDirectory.java | 7 +-
.../apache/lucene/misc/store/WindowsDirectory.java | 14 ++--
lucene/misc/src/java/overview.html | 23 +++---
.../lucene/misc/store/NativeLibEnableRule.java | 54 ++++++++++++++
.../lucene/misc/store/NativeUnixDirectoryTest.java | 46 ++++++++++++
.../lucene/misc/store/WindowsDirectoryTest.java | 40 +++++++++++
lucene/packaging/build.gradle | 4 +-
settings.gradle | 3 +-
17 files changed, 348 insertions(+), 47 deletions(-)
diff --git a/build.gradle b/build.gradle
index aaf5848..59cddf5 100644
--- a/build.gradle
+++ b/build.gradle
@@ -154,6 +154,9 @@ apply from: file('gradle/generation/snowball.gradle')
apply from: file('gradle/generation/kuromoji.gradle')
apply from: file('gradle/generation/nori.gradle')
+// Shared configuration of subprojects containing native code.
+apply from: file('gradle/native/disable-native.gradle')
+
// Additional development aids.
apply from: file('gradle/maven/maven-local.gradle')
apply from: file('gradle/testing/per-project-summary.gradle')
diff --git a/gradle/native/disable-native.gradle b/gradle/native/disable-native.gradle
new file mode 100644
index 0000000..beb977a
--- /dev/null
+++ b/gradle/native/disable-native.gradle
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This is the master switch to disable all tasks that compile
+// native (cpp) code.
+rootProject.ext {
+ buildNative = propertyOrDefault("build.native", true).toBoolean()
+}
+
+// Explicitly list all projects that should be configured for native extensions.
+// We could scan for projects with a the cpp-library plugin but this is faster.
+def nativeProjects = allprojects.findAll {it.path in [
+ ":lucene:misc:native"
+]}
+
+def javaProjectsWithNativeDeps = allprojects.findAll {it.path in [
+ ":lucene:misc"
+]}
+
+// Set up defaults for projects with native dependencies.
+configure(javaProjectsWithNativeDeps, {
+ configurations {
+ nativeDeps {
+ attributes {
+ attributes.attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage.class, Usage.NATIVE_RUNTIME))
+ attributes.attribute(CppBinary.OPTIMIZED_ATTRIBUTE, false)
+ }
+ }
+ }
+
+ plugins.withType(JavaPlugin) {
+ ext {
+ testOptions += [
+ [propName: 'tests.native', value: buildNative, description: "Enable tests that require native extensions."]
+ ]
+
+ nativeDepsDir = file("${buildDir}/nativeDeps")
+ }
+
+ // Only copy and attach native deps if native build is enabled.
+ if (buildNative) {
+ task copyNativeDeps(type: Sync) {
+ from configurations.nativeDeps
+ into nativeDepsDir
+ }
+
+ tasks.withType(Test) {
+ dependsOn copyNativeDeps
+ systemProperty "java.library.path", nativeDepsDir
+ }
+ }
+ }
+})
+
+// If native build is disabled we just disable all tasks in the active task set that
+// originate from "native" projects.
+//
+// Perhaps there is a cleaner way to do it but removing their references from
+// settings.gradle would remove them from IDE detection, dependency resolution, etc.
+// This way seems better.
+if (!buildNative) {
+ gradle.taskGraph.whenReady { taskGraph ->
+ def tasks = taskGraph.getAllTasks()
+ tasks.findAll { task -> task.project in nativeProjects }.each { task ->
+ task.enabled = false
+ }
+ }
+}
diff --git a/gradle/testing/randomization/policies/tests.policy b/gradle/testing/randomization/policies/tests.policy
index c6f3f4b..66b0dea 100644
--- a/gradle/testing/randomization/policies/tests.policy
+++ b/gradle/testing/randomization/policies/tests.policy
@@ -62,6 +62,10 @@ grant {
permission java.lang.RuntimePermission "getClassLoader";
permission java.lang.RuntimePermission "setContextClassLoader";
+ // Needed for loading native library (lucene:misc:native) in lucene:misc
+ permission java.lang.RuntimePermission "loadLibrary.LuceneNativeIO";
+ permission java.lang.RuntimePermission "writeFileDescriptor";
+
// TestLockFactoriesMultiJVM opens a random port on 127.0.0.1 (port 0 = ephemeral port range):
permission java.net.SocketPermission "127.0.0.1:0", "accept,listen,resolve";
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 26f3585..0fc90b4 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -18,6 +18,10 @@ System Requirements
API Changes
+* LUCENE-8982: Separate out native code to another module to allow cpp
+ build with gradle. This also changes the name of the native "posix-support"
+ library to LuceneNativeIO. (Zachary Chen, Dawid Weiss)
+
* LUCENE-9562: All binary analysis packages (and corresponding
Maven artifacts) with names containing '-analyzers-' have been renamed
to '-analysis-'. (Dawid Weiss)
diff --git a/lucene/misc/build.gradle b/lucene/misc/build.gradle
index 11758f7..efed4f2 100644
--- a/lucene/misc/build.gradle
+++ b/lucene/misc/build.gradle
@@ -22,4 +22,6 @@ description = 'Index tools and other miscellaneous code'
dependencies {
api project(':lucene:core')
testImplementation project(':lucene:test-framework')
-}
+
+ nativeDeps project(":lucene:misc:native")
+}
\ No newline at end of file
diff --git a/lucene/misc/native/build.gradle b/lucene/misc/native/build.gradle
new file mode 100644
index 0000000..3673660
--- /dev/null
+++ b/lucene/misc/native/build.gradle
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * This gets separated out from misc module into a native module due to incompatibility between cpp-library and java-library plugins.
+ * For details, please see https://github.com/gradle/gradle-native/issues/352#issuecomment-461724948
+ */
+import org.apache.tools.ant.taskdefs.condition.Os
+
+description = 'Module for native code'
+
+apply plugin: 'cpp-library'
+
+library {
+ baseName = 'LuceneNativeIO'
+
+ // Native build for Windows platform will be added in later stage
+ targetMachines = [
+ machines.linux.x86_64,
+ machines.macOS.x86_64,
+ machines.windows.x86_64
+ ]
+
+ // Point at platform-specific sources. Other platforms will be ignored
+ // (plugin won't find the toolchain).
+ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+ source.from file("${projectDir}/src/main/windows")
+ } else if (Os.isFamily(Os.FAMILY_UNIX) || Os.isFamily(Os.FAMILY_MAC)) {
+ source.from file("${projectDir}/src/main/posix")
+ }
+}
+
+tasks.withType(CppCompile).configureEach {
+ def javaHome = rootProject.ext.runtimeJava.getInstallationDirectory().getAsFile().getPath()
+
+ // Assume standard openjdk layout. This means only one architecture-specific include folder
+ // is present.
+ systemIncludes.from file("${javaHome}/include")
+
+ for (def path : [
+ file("${javaHome}/include/win32"),
+ file("${javaHome}/include/darwin"),
+ file("${javaHome}/include/linux"),
+ file("${javaHome}/include/solaris")]) {
+ if (path.exists()) {
+ systemIncludes.from path
+ }
+ }
+
+ compilerArgs.add '-fPIC'
+}
+
+tasks.withType(LinkSharedLibrary).configureEach {
+ linkerArgs.add '-lstdc++'
+}
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/store/NativePosixUtil.cpp b/lucene/misc/native/src/main/posix/NativePosixUtil.cpp
similarity index 87%
rename from lucene/misc/src/java/org/apache/lucene/misc/store/NativePosixUtil.cpp
rename to lucene/misc/native/src/main/posix/NativePosixUtil.cpp
index 999e6a2..55d6ef4 100644
--- a/lucene/misc/src/java/org/apache/lucene/misc/store/NativePosixUtil.cpp
+++ b/lucene/misc/native/src/main/posix/NativePosixUtil.cpp
@@ -38,12 +38,12 @@
#ifdef LINUX
/*
- * Class: org_apache_lucene_store_NativePosixUtil
+ * Class: org_apache_lucene_misc_store_NativePosixUtil
* Method: posix_fadvise
* Signature: (Ljava/io/FileDescriptor;JJI)V
*/
extern "C"
-JNIEXPORT jint JNICALL Java_org_apache_lucene_store_NativePosixUtil_posix_1fadvise(JNIEnv *env, jclass _ignore, jobject fileDescriptor, jlong offset, jlong len, jint advice)
+JNIEXPORT jint JNICALL Java_org_apache_lucene_misc_store_NativePosixUtil_posix_1fadvise(JNIEnv *env, jclass _ignore, jobject fileDescriptor, jlong offset, jlong len, jint advice)
{
jfieldID field_fd;
jmethodID const_fdesc;
@@ -103,12 +103,12 @@ JNIEXPORT jint JNICALL Java_org_apache_lucene_store_NativePosixUtil_posix_1fadvi
#endif
/*
- * Class: org_apache_lucene_store_NativePosixUtil
+ * Class: org_apache_lucene_misc_store_NativePosixUtil
* Method: open_direct
* Signature: (Ljava/lang/String;Z)Ljava/io/FileDescriptor;
*/
extern "C"
-JNIEXPORT jobject JNICALL Java_org_apache_lucene_store_NativePosixUtil_open_1direct(JNIEnv *env, jclass _ignore, jstring filename, jboolean readOnly)
+JNIEXPORT jobject JNICALL Java_org_apache_lucene_misc_store_NativePosixUtil_open_1direct(JNIEnv *env, jclass _ignore, jstring filename, jboolean readOnly)
{
jfieldID field_fd;
jmethodID const_fdesc;
@@ -169,12 +169,12 @@ JNIEXPORT jobject JNICALL Java_org_apache_lucene_store_NativePosixUtil_open_1dir
}
/*
- * Class: org_apache_lucene_store_NativePosixUtil
+ * Class: org_apache_lucene_misc_store_NativePosixUtil
* Method: pread
* Signature: (Ljava/io/FileDescriptor;JLjava/nio/ByteBuffer;)I
*/
extern "C"
-JNIEXPORT jlong JNICALL Java_org_apache_lucene_store_NativePosixUtil_pread(JNIEnv *env, jclass _ignore, jobject jfd, jlong pos, jobject byteBuf)
+JNIEXPORT jlong JNICALL Java_org_apache_lucene_misc_store_NativePosixUtil_pread(JNIEnv *env, jclass _ignore, jobject jfd, jlong pos, jobject byteBuf)
{
// get int fd:
jclass class_fdesc = env->FindClass("java/io/FileDescriptor");
@@ -214,12 +214,12 @@ JNIEXPORT jlong JNICALL Java_org_apache_lucene_store_NativePosixUtil_pread(JNIEn
}
/*
- * Class: org_apache_lucene_store_NativePosixUtil
+ * Class: org_apache_lucene_misc_store_NativePosixUtil
* Method: posix_madvise
* Signature: (Ljava/nio/ByteBuffer;I)I
*/
extern "C"
-JNIEXPORT jint JNICALL Java_org_apache_lucene_store_NativePosixUtil_posix_1madvise(JNIEnv *env, jclass _ignore, jobject buffer, jint advice) {
+JNIEXPORT jint JNICALL Java_org_apache_lucene_misc_store_NativePosixUtil_posix_1madvise(JNIEnv *env, jclass _ignore, jobject buffer, jint advice) {
void *p = env->GetDirectBufferAddress(buffer);
if (p == NULL) {
return -1;
@@ -280,12 +280,12 @@ JNIEXPORT jint JNICALL Java_org_apache_lucene_store_NativePosixUtil_posix_1madvi
/*
- * Class: org_apache_lucene_store_NativePosixUtil
+ * Class: org_apache_lucene_misc_store_NativePosixUtil
* Method: madvise
* Signature: (Ljava/nio/ByteBuffer;I)I
*/
extern "C"
-JNIEXPORT jint JNICALL Java_org_apache_lucene_store_NativePosixUtil_madvise(JNIEnv *env, jclass _ignore, jobject buffer, jint advice) {
+JNIEXPORT jint JNICALL Java_org_apache_lucene_misc_store_NativePosixUtil_madvise(JNIEnv *env, jclass _ignore, jobject buffer, jint advice) {
void *p = env->GetDirectBufferAddress(buffer);
if (p == NULL) {
return -1;
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.cpp b/lucene/misc/native/src/main/windows/WindowsDirectory.cpp
similarity index 88%
rename from lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.cpp
rename to lucene/misc/native/src/main/windows/WindowsDirectory.cpp
index 416aed9..20f79fe 100644
--- a/lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.cpp
+++ b/lucene/misc/native/src/main/windows/WindowsDirectory.cpp
@@ -56,11 +56,11 @@ void throwException(JNIEnv *env, const char *clazz, const char *msg)
/**
* Opens a handle to a file.
*
- * Class: org_apache_lucene_store_WindowsDirectory
+ * Class: org_apache_lucene_misc_store_WindowsDirectory
* Method: open
* Signature: (Ljava/lang/String;)J
*/
-JNIEXPORT jlong JNICALL Java_org_apache_lucene_store_WindowsDirectory_open
+JNIEXPORT jlong JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_open
(JNIEnv *env, jclass ignored, jstring filename)
{
char *fname;
@@ -95,11 +95,11 @@ JNIEXPORT jlong JNICALL Java_org_apache_lucene_store_WindowsDirectory_open
* Reads data into the byte array, starting at offset, for length characters.
* The read is positioned at pos.
*
- * Class: org_apache_lucene_store_WindowsDirectory
+ * Class: org_apache_lucene_misc_store_WindowsDirectory
* Method: read
* Signature: (J[BIIJ)I
*/
-JNIEXPORT jint JNICALL Java_org_apache_lucene_store_WindowsDirectory_read
+JNIEXPORT jint JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_read
(JNIEnv *env, jclass ignored, jlong fd, jbyteArray bytes, jint offset, jint length, jlong pos)
{
OVERLAPPED io = { 0 };
@@ -140,11 +140,11 @@ JNIEXPORT jint JNICALL Java_org_apache_lucene_store_WindowsDirectory_read
/**
* Closes a handle to a file
*
- * Class: org_apache_lucene_store_WindowsDirectory
+ * Class: org_apache_lucene_misc_store_WindowsDirectory
* Method: close
* Signature: (J)V
*/
-JNIEXPORT void JNICALL Java_org_apache_lucene_store_WindowsDirectory_close
+JNIEXPORT void JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_close
(JNIEnv *env, jclass ignored, jlong fd)
{
if (!CloseHandle((HANDLE) fd)) {
@@ -155,11 +155,11 @@ JNIEXPORT void JNICALL Java_org_apache_lucene_store_WindowsDirectory_close
/**
* Returns the length in bytes of a file.
*
- * Class: org_apache_lucene_store_WindowsDirectory
+ * Class: org_apache_lucene_misc_store_WindowsDirectory
* Method: length
* Signature: (J)J
*/
-JNIEXPORT jlong JNICALL Java_org_apache_lucene_store_WindowsDirectory_length
+JNIEXPORT jlong JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_length
(JNIEnv *env, jclass ignored, jlong fd)
{
BY_HANDLE_FILE_INFORMATION info;
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/store/NativePosixUtil.java b/lucene/misc/src/java/org/apache/lucene/misc/store/NativePosixUtil.java
index d13dd4b..c119e52 100644
--- a/lucene/misc/src/java/org/apache/lucene/misc/store/NativePosixUtil.java
+++ b/lucene/misc/src/java/org/apache/lucene/misc/store/NativePosixUtil.java
@@ -33,7 +33,7 @@ public final class NativePosixUtil {
public final static int NOREUSE = 5;
static {
- System.loadLibrary("NativePosixUtil");
+ System.loadLibrary("LuceneNativeIO");
}
private static native int posix_fadvise(FileDescriptor fd, long offset, long len, int advise) throws IOException;
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/store/NativeUnixDirectory.java b/lucene/misc/src/java/org/apache/lucene/misc/store/NativeUnixDirectory.java
index 8511754..5057a0a 100644
--- a/lucene/misc/src/java/org/apache/lucene/misc/store/NativeUnixDirectory.java
+++ b/lucene/misc/src/java/org/apache/lucene/misc/store/NativeUnixDirectory.java
@@ -56,10 +56,9 @@ import org.apache.lucene.util.SuppressForbidden;
*
* <p>To use this you must compile
* NativePosixUtil.cpp (exposes Linux-specific APIs through
- * JNI) for your platform, by running <code>ant
- * build-native-unix</code>, and then putting the resulting
- * <code>libNativePosixUtil.so</code> (from
- * <code>lucene/build/native</code>) onto your dynamic
+ * JNI) for your platform, by running <code>./gradlew build</code>, and then putting the resulting
+ * <code>libLuceneNativeIO.so</code> or <code>libLuceneNativeIO.dylib</code>
+ * (from <code>lucene/misc/native/build/lib/release/platform/</code>) onto your dynamic
* linker search path.
*
* <p><b>WARNING</b>: this code is very new and quite easily
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.java b/lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.java
index 4f96c82..d791eca 100644
--- a/lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.java
+++ b/lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.java
@@ -34,14 +34,10 @@ import java.nio.file.Path;
* <p>
* Steps:
* <ol>
- * <li>Compile the source code to create WindowsDirectory.dll:
- * <blockquote>
- * c:\mingw\bin\g++ -Wall -D_JNI_IMPLEMENTATION_ -Wl,--kill-at
- * -I"%JAVA_HOME%\include" -I"%JAVA_HOME%\include\win32" -static-libgcc
- * -static-libstdc++ -shared WindowsDirectory.cpp -o WindowsDirectory.dll
- * </blockquote>
- * For 64-bit JREs, use mingw64, with the -m64 option.
- * <li>Put WindowsDirectory.dll into some directory in your windows PATH
+ * <li>Compile the source code to create libLuceneNativeIO.dll: <code>./gradlew build</code>
+ * <li>Put the resulting <code>libLuceneNativeIO.dll</code>
+ * (from <code>lucene/misc/native/build/lib/release/platform/</code>)
+ * into some directory in your windows PATH
* <li>Open indexes with WindowsDirectory and use it.
* </ol>
* @lucene.experimental
@@ -50,7 +46,7 @@ public class WindowsDirectory extends FSDirectory {
private static final int DEFAULT_BUFFERSIZE = 4096; /* default pgsize on ia32/amd64 */
static {
- System.loadLibrary("WindowsDirectory");
+ System.loadLibrary("LuceneNativeIO");
}
/** Create a new WindowsDirectory for the named location.
diff --git a/lucene/misc/src/java/overview.html b/lucene/misc/src/java/overview.html
index 1dec99f..5cda451 100644
--- a/lucene/misc/src/java/overview.html
+++ b/lucene/misc/src/java/overview.html
@@ -42,26 +42,25 @@ used during merging of segments larger than a specified size (default
searching, keeping search more responsive while large merges run.
<p>
-See <a target=_top href="http://blog.mikemccandless.com/2010/06/lucene-and-fadvisemadvise.html">this blog post</a>
+See <a target="_top" href="http://blog.mikemccandless.com/2010/06/lucene-and-fadvisemadvise.html">this blog post</a>
for details.
-Steps to build:
+<p>Steps to build (from the project's root directory):
<ul>
- <li> <code>cd lucene/misc/</code>
-
- <li> To compile NativePosixUtil.cpp -> libNativePosixUtil.so, run<code> ant build-native-unix</code>.
+ <li>Compile both the native library part (<code>libLuceneNativeIO</code>) and Java sources with:
+ <code>./gradlew -p lucene/misc build</code>.</li>
- <li><code>libNativePosixUtil.so</code> will be located in the <code>lucene/build/native/</code> folder
-
- <li> Make sure libNativePosixUtil.so is on your LD_LIBRARY_PATH so java can find it (something like <code>export LD_LIBRARY_PATH=/path/to/dir:$LD_LIBRARY_PATH</code>, where /path/to/dir contains libNativePosixUtil.so)
+ <li>The native library will be located in the <code>lucene/misc/native/build/lib/main/release/<i>your-platform</i></code> folder.</li>
- <li> <code>ant jar</code> to compile the java source and put that JAR on your CLASSPATH
+ <li>On Unix-ish systems, make sure <code>libNativePosixUtil.so</code> is on your
+ <code>LD_LIBRARY_PATH</code> so java can find it (something like <code>export LD_LIBRARY_PATH=/path/to/dir:$LD_LIBRARY_PATH</code>,
+ where /path/to/dir contains <code>libLuceneNativeIO.so</code>).</li>
</ul>
<p>
-NativePosixUtil.cpp/java also expose access to the posix_madvise,
-madvise, posix_fadvise functions, which are somewhat more cross
-platform than O_DIRECT, however, in testing (see above link), these
+The native library exposes access to the <code>posix_madvise</code>,
+<code>madvise</code>, <code>posix_fadvise</code> functions, which are somewhat more cross
+platform than <code>O_DIRECT</code>, however, in testing (see above link), these
APIs did not seem to help prevent buffer cache eviction.
</body>
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/NativeLibEnableRule.java b/lucene/misc/src/test/org/apache/lucene/misc/store/NativeLibEnableRule.java
new file mode 100644
index 0000000..33c4b57
--- /dev/null
+++ b/lucene/misc/src/test/org/apache/lucene/misc/store/NativeLibEnableRule.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.misc.store;
+
+import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
+import org.apache.lucene.util.Constants;
+import org.junit.Assume;
+
+import java.util.Set;
+
+public class NativeLibEnableRule extends TestRuleAdapter {
+ enum OperatingSystem {
+ LINUX(Constants.LINUX),
+ WINDOWS(Constants.WINDOWS),
+ SUN_OS(Constants.SUN_OS),
+ MAC(Constants.MAC_OS_X),
+ FREE_BSD(Constants.FREE_BSD);
+
+ public final boolean enabled;
+
+ OperatingSystem(boolean enabled) {
+ this.enabled = enabled;
+ }
+ }
+
+ private final Set<OperatingSystem> runOn;
+
+ public NativeLibEnableRule(Set<OperatingSystem> runOn) {
+ this.runOn = runOn;
+ }
+
+ @Override
+ protected void before() {
+ Assume.assumeTrue("Test ignored (tests.native is false)",
+ Boolean.parseBoolean(System.getProperty("tests.native", "false")));
+
+ Assume.assumeTrue("Test ignored, only applies to architectures: " + runOn,
+ runOn.stream().anyMatch(os -> os.enabled));
+ }
+}
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/NativeUnixDirectoryTest.java b/lucene/misc/src/test/org/apache/lucene/misc/store/NativeUnixDirectoryTest.java
new file mode 100644
index 0000000..d275950
--- /dev/null
+++ b/lucene/misc/src/test/org/apache/lucene/misc/store/NativeUnixDirectoryTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.misc.store;
+
+import com.carrotsearch.randomizedtesting.LifecycleScope;
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.MergeInfo;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Rule;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+import java.util.EnumSet;
+
+public class NativeUnixDirectoryTest extends LuceneTestCase {
+ @Rule
+ public static TestRule requiresNative = new NativeLibEnableRule(
+ EnumSet.of(NativeLibEnableRule.OperatingSystem.MAC,
+ NativeLibEnableRule.OperatingSystem.FREE_BSD,
+ NativeLibEnableRule.OperatingSystem.LINUX));
+
+ public void testLibraryLoaded() throws IOException {
+ try (ByteBuffersDirectory ramDir = new ByteBuffersDirectory();
+ Directory dir = new NativeUnixDirectory(RandomizedTest.newTempDir(LifecycleScope.TEST), ramDir)) {
+ MergeInfo mergeInfo = new MergeInfo(1000, Integer.MAX_VALUE, true, 1);
+ dir.createOutput("test", new IOContext(mergeInfo)).close();
+ }
+ }
+}
\ No newline at end of file
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/WindowsDirectoryTest.java b/lucene/misc/src/test/org/apache/lucene/misc/store/WindowsDirectoryTest.java
new file mode 100644
index 0000000..2323c2a
--- /dev/null
+++ b/lucene/misc/src/test/org/apache/lucene/misc/store/WindowsDirectoryTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.misc.store;
+
+import com.carrotsearch.randomizedtesting.LifecycleScope;
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Rule;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+import java.util.EnumSet;
+
+public class WindowsDirectoryTest extends LuceneTestCase {
+ @Rule
+ public static TestRule requiresNative = new NativeLibEnableRule(
+ EnumSet.of(NativeLibEnableRule.OperatingSystem.WINDOWS));
+
+ public void testLibraryLoaded() throws IOException {
+ try (Directory dir = new WindowsDirectory(RandomizedTest.newTempDir(LifecycleScope.TEST))) {
+ dir.createOutput("test", IOContext.DEFAULT).close();
+ }
+ }
+}
\ No newline at end of file
diff --git a/lucene/packaging/build.gradle b/lucene/packaging/build.gradle
index 151e6d5..1c4aa3e 100644
--- a/lucene/packaging/build.gradle
+++ b/lucene/packaging/build.gradle
@@ -32,7 +32,9 @@ def includeInBinaries = project(":lucene").subprojects.findAll {subproject ->
":lucene:packaging",
":lucene:documentation",
// Exclude parent container project of analysis modules (no artifacts).
- ":lucene:analysis"
+ ":lucene:analysis",
+ // Exclude native module, which requires manual copying and enabling
+ ":lucene:misc:native"
])
}
diff --git a/settings.gradle b/settings.gradle
index 61cf671..be2c09c 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -42,6 +42,7 @@ include "lucene:join"
include "lucene:luke"
include "lucene:memory"
include "lucene:misc"
+include "lucene:misc:native"
include "lucene:monitor"
include "lucene:queries"
include "lucene:queryparser"
@@ -73,4 +74,4 @@ include "solr:example"
include "solr:documentation"
include "solr:packaging"
include "solr:docker"
-include "solr:docker:package"
\ No newline at end of file
+include "solr:docker:package"