You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2022/09/15 14:22:52 UTC
[lucene] branch main updated: GH-11172: remove WindowsDirectory and native subproject. (#11774)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 9acc6539959 GH-11172: remove WindowsDirectory and native subproject. (#11774)
9acc6539959 is described below
commit 9acc6539959f79c42001f7d95afa9e65b53e7894
Author: Dawid Weiss <da...@carrotsearch.com>
AuthorDate: Thu Sep 15 16:22:46 2022 +0200
GH-11172: remove WindowsDirectory and native subproject. (#11774)
---
build.gradle | 3 -
gradle/native/disable-native.gradle | 84 ----------
lucene/CHANGES.txt | 3 +
lucene/misc/build.gradle | 2 -
lucene/misc/native/build.gradle | 60 -------
.../native/src/main/windows/WindowsDirectory.cpp | 175 ---------------------
.../apache/lucene/misc/store/WindowsDirectory.java | 147 -----------------
.../lucene/misc/store/NativeLibEnableRule.java | 55 -------
.../lucene/misc/store/TestWindowsDirectory.java | 39 -----
settings.gradle | 1 -
10 files changed, 3 insertions(+), 566 deletions(-)
diff --git a/build.gradle b/build.gradle
index dd5f63365ef..fedea545eea 100644
--- a/build.gradle
+++ b/build.gradle
@@ -158,9 +158,6 @@ apply from: file('gradle/generation/antlr.gradle')
apply from: file('gradle/datasets/external-datasets.gradle')
-// Shared configuration of subprojects containing native code.
-apply from: file('gradle/native/disable-native.gradle')
-
// Additional development aids.
apply from: file('gradle/testing/per-project-summary.gradle')
apply from: file('gradle/testing/slowest-tests-at-end.gradle')
diff --git a/gradle/native/disable-native.gradle b/gradle/native/disable-native.gradle
deleted file mode 100644
index 3cc239dbacb..00000000000
--- a/gradle/native/disable-native.gradle
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.tools.ant.taskdefs.condition.Os
-
-// This is the master switch to disable all tasks that compile
-// native (cpp) code.
-rootProject.ext {
- buildNative = propertyOrDefault("build.native", false).toBoolean()
-}
-
-// Explicitly list all projects that should be configured for native extensions.
-// We could scan for projects with a the cpp-library plugin but this is faster.
-def nativeProjects = allprojects.findAll {it.path in [
- ":lucene:misc:native"
-]}
-
-def javaProjectsWithNativeDeps = allprojects.findAll {it.path in [
- ":lucene:misc"
-]}
-
-// Set up defaults for projects with native dependencies.
-configure(javaProjectsWithNativeDeps, {
- configurations {
- nativeDeps {
- attributes {
- attributes.attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage.class, Usage.NATIVE_RUNTIME))
- attributes.attribute(CppBinary.OPTIMIZED_ATTRIBUTE, false)
- }
- }
- }
-
- plugins.withType(JavaPlugin) {
- ext {
- testOptions += [
- [propName: 'tests.native', value: buildNative, description: "Enable tests that require native extensions."]
- ]
-
- nativeDepsDir = file("${buildDir}/nativeDeps")
- }
-
- // Only copy and attach native deps if native build is enabled.
- if (buildNative && Os.isFamily(Os.FAMILY_WINDOWS)) {
- task copyNativeDeps(type: Sync) {
- from configurations.nativeDeps
- into nativeDepsDir
- }
-
- tasks.withType(Test) {
- dependsOn copyNativeDeps
- systemProperty "java.library.path", nativeDepsDir
- }
- }
- }
-})
-
-// If native build is disabled we just disable all tasks in the active task set that
-// originate from "native" projects.
-//
-// Perhaps there is a cleaner way to do it but removing their references from
-// settings.gradle would remove them from IDE detection, dependency resolution, etc.
-// This way seems better.
-if (!buildNative) {
- gradle.taskGraph.whenReady { taskGraph ->
- def tasks = taskGraph.getAllTasks()
- tasks.findAll { task -> task.project in nativeProjects }.each { task ->
- task.enabled = false
- }
- }
-}
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d717c44bfa2..7b2e50dcbcf 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -90,6 +90,9 @@ API Changes
* GITHUB#11742: MatchingFacetSetsCounts#getTopChildren now properly returns "top" children instead
of all children. (Greg Miller)
+* GITHUB#11772: Removed native subproject and WindowsDirectory implementation from lucene.misc. Recommendation:
+ use MMapDirectory implementation on Windows. (Robert Muir, Uwe Schindler, Dawid Weiss)
+
Bug Fixes
---------------------
* GITHUB#11726: Indexing term vectors on large documents could fail due to
diff --git a/lucene/misc/build.gradle b/lucene/misc/build.gradle
index 1c4468a504b..a21dfeaafa7 100644
--- a/lucene/misc/build.gradle
+++ b/lucene/misc/build.gradle
@@ -22,6 +22,4 @@ description = 'Index tools and other miscellaneous code'
dependencies {
moduleApi project(':lucene:core')
moduleTestImplementation project(':lucene:test-framework')
-
- nativeDeps project(":lucene:misc:native")
}
\ No newline at end of file
diff --git a/lucene/misc/native/build.gradle b/lucene/misc/native/build.gradle
deleted file mode 100644
index 664410ed903..00000000000
--- a/lucene/misc/native/build.gradle
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * This gets separated out from misc module into a native module due to incompatibility between cpp-library and java-library plugins.
- * For details, please see https://github.com/gradle/gradle-native/issues/352#issuecomment-461724948
- */
-import org.apache.tools.ant.taskdefs.condition.Os
-
-description = 'Module for native code'
-
-apply plugin: 'cpp-library'
-
-library {
- baseName = 'LuceneNativeIO'
-
- // Native build for Windows platform will be added in later stage
- targetMachines = [
- machines.windows.x86_64
- ]
-
- // Point at platform-specific sources. Other platforms will be ignored
- // (plugin won't find the toolchain).
- if (Os.isFamily(Os.FAMILY_WINDOWS)) {
- source.from file("${projectDir}/src/main/windows")
- }
-}
-
-tasks.withType(CppCompile).configureEach {
- def javaHome = rootProject.ext.runtimeJavaHome
-
- // Assume standard openjdk layout. This means only one architecture-specific include folder is present.
- systemIncludes.from file("${javaHome}/include")
-
- for (def path : [file("${javaHome}/include/win32")]) {
- if (path.exists()) {
- systemIncludes.from path
- }
- }
-
- compilerArgs.add '-fPIC'
-}
-
-tasks.withType(LinkSharedLibrary).configureEach {
- linkerArgs.add '-lstdc++'
-}
diff --git a/lucene/misc/native/src/main/windows/WindowsDirectory.cpp b/lucene/misc/native/src/main/windows/WindowsDirectory.cpp
deleted file mode 100644
index 20f79fe1954..00000000000
--- a/lucene/misc/native/src/main/windows/WindowsDirectory.cpp
+++ /dev/null
@@ -1,175 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-#include <jni.h>
-#include "windows.h"
-
-/**
- * Windows Native IO methods.
- */
-extern "C" {
-
-/**
- * Utility to format a Windows system error code into an exception.
- */
-void throwIOException(JNIEnv *env, DWORD error)
-{
- jclass ioex;
- char *msg;
-
- ioex = env->FindClass("java/io/IOException");
-
- if (ioex != NULL) {
- FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
- NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR) &msg, 0, NULL );
- env->ThrowNew(ioex, msg);
- LocalFree(msg);
- }
-}
-
-/**
- * Utility to throw Exceptions on bad input
- */
-void throwException(JNIEnv *env, const char *clazz, const char *msg)
-{
- jclass exc = env->FindClass(clazz);
-
- if (exc != NULL) {
- env->ThrowNew(exc, msg);
- }
-}
-
-/**
- * Opens a handle to a file.
- *
- * Class: org_apache_lucene_misc_store_WindowsDirectory
- * Method: open
- * Signature: (Ljava/lang/String;)J
- */
-JNIEXPORT jlong JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_open
- (JNIEnv *env, jclass ignored, jstring filename)
-{
- char *fname;
- HANDLE handle;
-
- if (filename == NULL) {
- throwException(env, "java/lang/NullPointerException", "filename must not be null");
- return -1;
- }
-
- fname = (char *) env->GetStringUTFChars(filename, NULL);
-
- if (fname == NULL) {
- throwException(env, "java/lang/IllegalArgumentException", "invalid filename");
- return -1;
- }
-
- handle = CreateFile(fname, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE,
- NULL, OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS, NULL);
-
- env->ReleaseStringUTFChars(filename, fname);
-
- if (handle == INVALID_HANDLE_VALUE) {
- throwIOException(env, GetLastError());
- return -1;
- }
-
- return (jlong) handle;
-}
-
-/**
- * Reads data into the byte array, starting at offset, for length characters.
- * The read is positioned at pos.
- *
- * Class: org_apache_lucene_misc_store_WindowsDirectory
- * Method: read
- * Signature: (J[BIIJ)I
- */
-JNIEXPORT jint JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_read
- (JNIEnv *env, jclass ignored, jlong fd, jbyteArray bytes, jint offset, jint length, jlong pos)
-{
- OVERLAPPED io = { 0 };
- DWORD numRead = -1;
-
- io.Offset = (DWORD) (pos & 0xFFFFFFFF);
- io.OffsetHigh = (DWORD) ((pos >> 0x20) & 0x7FFFFFFF);
-
- if (bytes == NULL) {
- throwException(env, "java/lang/NullPointerException", "bytes must not be null");
- return -1;
- }
-
- if (length <= 4096) { /* For small buffers, avoid GetByteArrayElements' copy */
- char buffer[length];
-
- if (ReadFile((HANDLE) fd, &buffer, length, &numRead, &io)) {
- env->SetByteArrayRegion(bytes, offset, numRead, (const jbyte *) buffer);
- } else {
- throwIOException(env, GetLastError());
- numRead = -1;
- }
-
- } else {
- jbyte *buffer = env->GetByteArrayElements (bytes, NULL);
-
- if (!ReadFile((HANDLE) fd, (void *)(buffer+offset), length, &numRead, &io)) {
- throwIOException(env, GetLastError());
- numRead = -1;
- }
-
- env->ReleaseByteArrayElements(bytes, buffer, numRead == 0 || numRead == -1 ? JNI_ABORT : 0);
- }
-
- return numRead;
-}
-
-/**
- * Closes a handle to a file
- *
- * Class: org_apache_lucene_misc_store_WindowsDirectory
- * Method: close
- * Signature: (J)V
- */
-JNIEXPORT void JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_close
- (JNIEnv *env, jclass ignored, jlong fd)
-{
- if (!CloseHandle((HANDLE) fd)) {
- throwIOException(env, GetLastError());
- }
-}
-
-/**
- * Returns the length in bytes of a file.
- *
- * Class: org_apache_lucene_misc_store_WindowsDirectory
- * Method: length
- * Signature: (J)J
- */
-JNIEXPORT jlong JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_length
- (JNIEnv *env, jclass ignored, jlong fd)
-{
- BY_HANDLE_FILE_INFORMATION info;
-
- if (GetFileInformationByHandle((HANDLE) fd, (LPBY_HANDLE_FILE_INFORMATION) &info)) {
- return (jlong) (((DWORDLONG) info.nFileSizeHigh << 0x20) + info.nFileSizeLow);
- } else {
- throwIOException(env, GetLastError());
- return -1;
- }
-}
-
-} /* extern "C" */
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.java b/lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.java
deleted file mode 100644
index c41401af3c9..00000000000
--- a/lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.misc.store;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Path;
-import org.apache.lucene.store.BufferedIndexInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.FSLockFactory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.LockFactory;
-
-/**
- * Native {@link Directory} implementation for Microsoft Windows.
- *
- * <p>Steps:
- *
- * <ol>
- * <li>Compile the source code to create libLuceneNativeIO.dll: <code>./gradlew build</code>
- * <li>Put the resulting <code>libLuceneNativeIO.dll</code> (from <code>
- * lucene/misc/native/build/lib/release/platform/</code>) into some directory in your windows
- * PATH
- * <li>Open indexes with WindowsDirectory and use it.
- * </ol>
- *
- * @lucene.experimental
- */
-public class WindowsDirectory extends FSDirectory {
- private static final int DEFAULT_BUFFERSIZE = 4096; /* default pgsize on ia32/amd64 */
-
- static {
- System.loadLibrary("LuceneNativeIO");
- }
-
- /**
- * Create a new WindowsDirectory for the named location.
- *
- * @param path the path of the directory
- * @param lockFactory the lock factory to use
- * @throws IOException If there is a low-level I/O error
- */
- public WindowsDirectory(Path path, LockFactory lockFactory) throws IOException {
- super(path, lockFactory);
- }
-
- /**
- * Create a new WindowsDirectory for the named location and {@link FSLockFactory#getDefault()}.
- *
- * @param path the path of the directory
- * @throws IOException If there is a low-level I/O error
- */
- public WindowsDirectory(Path path) throws IOException {
- this(path, FSLockFactory.getDefault());
- }
-
- @Override
- public IndexInput openInput(String name, IOContext context) throws IOException {
- ensureOpen();
- return new WindowsIndexInput(
- getDirectory().resolve(name),
- Math.max(BufferedIndexInput.bufferSize(context), DEFAULT_BUFFERSIZE));
- }
-
- static class WindowsIndexInput extends BufferedIndexInput {
- private final long fd;
- private final long length;
- boolean isClone;
- boolean isOpen;
-
- public WindowsIndexInput(Path file, int bufferSize) throws IOException {
- super("WindowsIndexInput(path=\"" + file + "\")", bufferSize);
- fd = WindowsDirectory.open(file.toString());
- length = WindowsDirectory.length(fd);
- isOpen = true;
- }
-
- @Override
- protected void readInternal(ByteBuffer b) throws IOException {
- int bytesRead;
- try {
- bytesRead =
- WindowsDirectory.read(fd, b.array(), b.position(), b.remaining(), getFilePointer());
- } catch (IOException ioe) {
- throw new IOException(ioe.getMessage() + ": " + this, ioe);
- }
-
- if (bytesRead != length) {
- throw new EOFException("read past EOF: " + this);
- }
- }
-
- @Override
- protected void seekInternal(long pos) throws IOException {}
-
- @Override
- public synchronized void close() throws IOException {
- // NOTE: we synchronize and track "isOpen" because Lucene sometimes closes IIs twice!
- if (!isClone && isOpen) {
- WindowsDirectory.close(fd);
- isOpen = false;
- }
- }
-
- @Override
- public long length() {
- return length;
- }
-
- @Override
- public WindowsIndexInput clone() {
- WindowsIndexInput clone = (WindowsIndexInput) super.clone();
- clone.isClone = true;
- return clone;
- }
- }
-
- /** Opens a handle to a file. */
- private static native long open(String filename) throws IOException;
-
- /** Reads data from a file at pos into bytes */
- private static native int read(long fd, byte[] bytes, int offset, int length, long pos)
- throws IOException;
-
- /** Closes a handle to a file */
- private static native void close(long fd) throws IOException;
-
- /** Returns the length of a file */
- private static native long length(long fd) throws IOException;
-}
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/NativeLibEnableRule.java b/lucene/misc/src/test/org/apache/lucene/misc/store/NativeLibEnableRule.java
deleted file mode 100644
index 5586b4f5665..00000000000
--- a/lucene/misc/src/test/org/apache/lucene/misc/store/NativeLibEnableRule.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.misc.store;
-
-import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
-import java.util.Set;
-import org.apache.lucene.util.Constants;
-import org.junit.Assume;
-
-public class NativeLibEnableRule extends TestRuleAdapter {
- enum OperatingSystem {
- LINUX(Constants.LINUX),
- WINDOWS(Constants.WINDOWS),
- SUN_OS(Constants.SUN_OS),
- MAC(Constants.MAC_OS_X),
- FREE_BSD(Constants.FREE_BSD);
-
- public final boolean enabled;
-
- OperatingSystem(boolean enabled) {
- this.enabled = enabled;
- }
- }
-
- private final Set<OperatingSystem> runOn;
-
- public NativeLibEnableRule(Set<OperatingSystem> runOn) {
- this.runOn = runOn;
- }
-
- @Override
- protected void before() {
- Assume.assumeTrue(
- "Test ignored (tests.native is false)",
- Boolean.parseBoolean(System.getProperty("tests.native", "false")));
-
- Assume.assumeTrue(
- "Test ignored, only applies to architectures: " + runOn,
- runOn.stream().anyMatch(os -> os.enabled));
- }
-}
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/TestWindowsDirectory.java b/lucene/misc/src/test/org/apache/lucene/misc/store/TestWindowsDirectory.java
deleted file mode 100644
index c4326469f08..00000000000
--- a/lucene/misc/src/test/org/apache/lucene/misc/store/TestWindowsDirectory.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.misc.store;
-
-import com.carrotsearch.randomizedtesting.LifecycleScope;
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import java.io.IOException;
-import java.util.EnumSet;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.junit.Rule;
-import org.junit.rules.TestRule;
-
-public class TestWindowsDirectory extends LuceneTestCase {
- @Rule
- public static TestRule requiresNative =
- new NativeLibEnableRule(EnumSet.of(NativeLibEnableRule.OperatingSystem.WINDOWS));
-
- public void testLibraryLoaded() throws IOException {
- try (Directory dir = new WindowsDirectory(RandomizedTest.newTempDir(LifecycleScope.TEST))) {
- dir.createOutput("test", IOContext.DEFAULT).close();
- }
- }
-}
diff --git a/settings.gradle b/settings.gradle
index 971fbb6c5bc..4d22af5fa2e 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -55,7 +55,6 @@ include "lucene:join"
include "lucene:luke"
include "lucene:memory"
include "lucene:misc"
-include "lucene:misc:native"
include "lucene:monitor"
include "lucene:queries"
include "lucene:queryparser"