You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2023/11/03 19:57:23 UTC
(lucene) branch branch_9x updated: Refactor access to VM options and move some VM options to oal.util.Constants (#12754)
This is an automated email from the ASF dual-hosted git repository.
uschindler pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 71b3e4c97fb Refactor access to VM options and move some VM options to oal.util.Constants (#12754)
71b3e4c97fb is described below
commit 71b3e4c97fbc81a98ccb19511d42487206df8c03
Author: Uwe Schindler <us...@apache.org>
AuthorDate: Fri Nov 3 20:55:36 2023 +0100
Refactor access to VM options and move some VM options to oal.util.Constants (#12754)
---
lucene/CHANGES.txt | 3 +
.../vectorization/VectorizationProvider.java | 23 +-----
.../src/java/org/apache/lucene/util/Constants.java | 82 ++++++++++++++------
.../org/apache/lucene/util/HotspotVMOptions.java | 90 ++++++++++++++++++++++
.../org/apache/lucene/util/RamUsageEstimator.java | 62 ++-------------
.../vectorization/PanamaVectorUtilSupport.java | 34 +-------
.../vectorization/PanamaVectorizationProvider.java | 3 +-
7 files changed, 164 insertions(+), 133 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d2e4cfbcf15..2d76ff54249 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -73,6 +73,9 @@ Improvements
* GITHUB#12689: TaskExecutor to cancel all tasks on exception to avoid needless computation. (Luca Cavanna)
+* GITHUB#12754: Refactor lookup of Hotspot VM options and do not initialize constants with NULL
+ if SecurityManager prevents access. (Uwe Schindler)
+
Optimizations
---------------------
* GITHUB#12183: Make TermStates#build concurrent. (Shubham Chaudhary)
diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
index ed4066a94ac..3d565b650a9 100644
--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
@@ -21,8 +21,6 @@ import java.lang.Runtime.Version;
import java.lang.StackWalker.StackFrame;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
@@ -31,7 +29,7 @@ import java.util.Set;
import java.util.function.Predicate;
import java.util.logging.Logger;
import java.util.stream.Stream;
-import org.apache.lucene.util.SuppressForbidden;
+import org.apache.lucene.util.Constants;
import org.apache.lucene.util.VectorUtil;
/**
@@ -129,7 +127,7 @@ public abstract class VectorizationProvider {
"Vector bitsize and/or integer vectors enforcement; using default vectorization provider outside of testMode");
return new DefaultVectorizationProvider();
}
- if (isClientVM()) {
+ if (Constants.IS_CLIENT_VM) {
LOG.warning("C2 compiler is disabled; Java vector incubator API can't be enabled");
return new DefaultVectorizationProvider();
}
@@ -188,23 +186,6 @@ public abstract class VectorizationProvider {
&& !Objects.equals("I", "i".toUpperCase(Locale.getDefault()));
}
- @SuppressWarnings("removal")
- @SuppressForbidden(reason = "security manager")
- private static boolean isClientVM() {
- try {
- final PrivilegedAction<Boolean> action =
- () -> System.getProperty("java.vm.info", "").contains("emulated-client");
- return AccessController.doPrivileged(action);
- } catch (
- @SuppressWarnings("unused")
- SecurityException e) {
- LOG.warning(
- "SecurityManager denies permission to 'java.vm.info' system property, so state of C2 compiler can't be detected. "
- + "In case of performance issues allow access to this property.");
- return false;
- }
- }
-
// add all possible callers here as FQCN:
private static final Set<String> VALID_CALLERS = Set.of("org.apache.lucene.util.VectorUtil");
diff --git a/lucene/core/src/java/org/apache/lucene/util/Constants.java b/lucene/core/src/java/org/apache/lucene/util/Constants.java
index f38141c616a..b10b00b8904 100644
--- a/lucene/core/src/java/org/apache/lucene/util/Constants.java
+++ b/lucene/core/src/java/org/apache/lucene/util/Constants.java
@@ -16,15 +16,22 @@
*/
package org.apache.lucene.util;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Objects;
+import java.util.logging.Logger;
+
/** Some useful constants. */
public final class Constants {
private Constants() {} // can't construct
+ private static final String UNKNOWN = "Unknown";
+
/** JVM vendor info. */
- public static final String JVM_VENDOR = System.getProperty("java.vm.vendor");
+ public static final String JVM_VENDOR = getSysProp("java.vm.vendor", UNKNOWN);
/** JVM vendor name. */
- public static final String JVM_NAME = System.getProperty("java.vm.name");
+ public static final String JVM_NAME = getSysProp("java.vm.name", UNKNOWN);
/**
* Get the full version string of the current runtime.
@@ -51,7 +58,7 @@ public final class Constants {
@Deprecated public static final String JAVA_VERSION = System.getProperty("java.version");
/** The value of <code>System.getProperty("os.name")</code>. * */
- public static final String OS_NAME = System.getProperty("os.name");
+ public static final String OS_NAME = getSysProp("os.name", UNKNOWN);
/** True iff running on Linux. */
public static final boolean LINUX = OS_NAME.startsWith("Linux");
@@ -69,37 +76,68 @@ public final class Constants {
public static final boolean FREE_BSD = OS_NAME.startsWith("FreeBSD");
/** The value of <code>System.getProperty("os.arch")</code>. */
- public static final String OS_ARCH = System.getProperty("os.arch");
+ public static final String OS_ARCH = getSysProp("os.arch", UNKNOWN);
/** The value of <code>System.getProperty("os.version")</code>. */
- public static final String OS_VERSION = System.getProperty("os.version");
+ public static final String OS_VERSION = getSysProp("os.version", UNKNOWN);
/** The value of <code>System.getProperty("java.vendor")</code>. */
- public static final String JAVA_VENDOR = System.getProperty("java.vendor");
+ public static final String JAVA_VENDOR = getSysProp("java.vendor", UNKNOWN);
+
+ /** True iff the Java runtime is a client runtime and C2 compiler is not enabled */
+ public static final boolean IS_CLIENT_VM =
+ getSysProp("java.vm.info", "").contains("emulated-client");
/** True iff running on a 64bit JVM */
- public static final boolean JRE_IS_64BIT;
+ public static final boolean JRE_IS_64BIT = is64Bit();
+
+ /** true iff we know fast FMA is supported, to deliver less error */
+ public static final boolean HAS_FAST_FMA =
+ (IS_CLIENT_VM == false)
+ && Objects.equals(OS_ARCH, "amd64")
+ && HotspotVMOptions.get("UseFMA").map(Boolean::valueOf).orElse(false);
+
+ private static boolean is64Bit() {
+ final String datamodel = getSysProp("sun.arch.data.model");
+ if (datamodel != null) {
+ return datamodel.contains("64");
+ } else {
+ return (OS_ARCH != null && OS_ARCH.contains("64"));
+ }
+ }
- static {
- boolean is64Bit = false;
- String datamodel = null;
+ private static String getSysProp(String property) {
try {
- datamodel = System.getProperty("sun.arch.data.model");
- if (datamodel != null) {
- is64Bit = datamodel.contains("64");
- }
+ return doPrivileged(() -> System.getProperty(property));
} catch (
@SuppressWarnings("unused")
- SecurityException ex) {
+ SecurityException se) {
+ logSecurityWarning(property);
+ return null;
}
- if (datamodel == null) {
- if (OS_ARCH != null && OS_ARCH.contains("64")) {
- is64Bit = true;
- } else {
- is64Bit = false;
- }
+ }
+
+ private static String getSysProp(String property, String def) {
+ try {
+ return doPrivileged(() -> System.getProperty(property, def));
+ } catch (
+ @SuppressWarnings("unused")
+ SecurityException se) {
+ logSecurityWarning(property);
+ return def;
}
- JRE_IS_64BIT = is64Bit;
+ }
+
+ private static void logSecurityWarning(String property) {
+ var log = Logger.getLogger(Constants.class.getName());
+ log.warning("SecurityManager prevented access to system property: " + property);
+ }
+
+ // Extracted to a method to be able to apply the SuppressForbidden annotation
+ @SuppressWarnings("removal")
+ @SuppressForbidden(reason = "security manager")
+ private static <T> T doPrivileged(PrivilegedAction<T> action) {
+ return AccessController.doPrivileged(action);
}
/**
diff --git a/lucene/core/src/java/org/apache/lucene/util/HotspotVMOptions.java b/lucene/core/src/java/org/apache/lucene/util/HotspotVMOptions.java
new file mode 100644
index 00000000000..70f963e1b37
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/HotspotVMOptions.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util;
+
+import java.lang.reflect.Method;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.logging.Logger;
+
+/** Accessor to get Hotspot VM Options (if available). */
+final class HotspotVMOptions {
+ private HotspotVMOptions() {} // can't construct
+
+ /** True if the Java VM is based on Hotspot and has the Hotspot MX bean readable by Lucene */
+ public static final boolean IS_HOTSPOT;
+
+ /**
+ * Returns an optional with the value of a Hotspot VM option. If the VM option does not exist or
+ * is not readable, returns an empty optional.
+ */
+ public static Optional<String> get(String name) {
+ return ACCESSOR.apply(Objects.requireNonNull(name, "name"));
+ }
+
+ private static final String MANAGEMENT_FACTORY_CLASS = "java.lang.management.ManagementFactory";
+ private static final String HOTSPOT_BEAN_CLASS = "com.sun.management.HotSpotDiagnosticMXBean";
+ private static final Function<String, Optional<String>> ACCESSOR;
+
+ static {
+ boolean isHotspot = false;
+ Function<String, Optional<String>> accessor = name -> Optional.empty();
+ try {
+ final Class<?> beanClazz = Class.forName(HOTSPOT_BEAN_CLASS);
+ // we use reflection for this, because the management factory is not part
+ // of java.base module:
+ final Object hotSpotBean =
+ Class.forName(MANAGEMENT_FACTORY_CLASS)
+ .getMethod("getPlatformMXBean", Class.class)
+ .invoke(null, beanClazz);
+ if (hotSpotBean != null) {
+ final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class);
+ final Method getValueMethod = getVMOptionMethod.getReturnType().getMethod("getValue");
+ isHotspot = true;
+ accessor =
+ name -> {
+ try {
+ final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, name);
+ return Optional.of(getValueMethod.invoke(vmOption).toString());
+ } catch (@SuppressWarnings("unused")
+ ReflectiveOperationException
+ | RuntimeException e) {
+ return Optional.empty();
+ }
+ };
+ }
+ } catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {
+ isHotspot = false;
+ final Logger log = Logger.getLogger(HotspotVMOptions.class.getName());
+ final Module module = HotspotVMOptions.class.getModule();
+ final ModuleLayer layer = module.getLayer();
+ // classpath / unnamed module has no layer, so we need to check:
+ if (layer != null
+ && layer.findModule("jdk.management").map(module::canRead).orElse(false) == false) {
+ log.warning(
+ "Lucene cannot access JVM internals to optimize algorithms or calculate object sizes, unless the 'jdk.management' Java module "
+ + "is readable [please add 'jdk.management' to modular application either by command line or its module descriptor].");
+ } else {
+ log.warning(
+ "Lucene cannot optimize algorithms or calculate object sizes for JVMs that are not based on Hotspot or a compatible implementation.");
+ }
+ }
+ IS_HOTSPOT = isHotspot;
+ ACCESSOR = accessor;
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java b/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
index 899a5941ada..109d19377fb 100644
--- a/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
+++ b/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
@@ -18,7 +18,6 @@ package org.apache.lucene.util;
import java.lang.reflect.Array;
import java.lang.reflect.Field;
-import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.security.AccessControlException;
import java.security.AccessController;
@@ -30,7 +29,6 @@ import java.util.Collections;
import java.util.IdentityHashMap;
import java.util.Locale;
import java.util.Map;
-import java.util.logging.Logger;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.Query;
@@ -112,64 +110,16 @@ public final class RamUsageEstimator {
/** For testing only */
static final boolean JVM_IS_HOTSPOT_64BIT;
- static final String MANAGEMENT_FACTORY_CLASS = "java.lang.management.ManagementFactory";
- static final String HOTSPOT_BEAN_CLASS = "com.sun.management.HotSpotDiagnosticMXBean";
-
/** Initialize constants and try to collect information about the JVM internals. */
static {
- if (Constants.JRE_IS_64BIT) {
+ if (Constants.JRE_IS_64BIT && HotspotVMOptions.IS_HOTSPOT) {
// Try to get compressed oops and object alignment (the default seems to be 8 on Hotspot);
// (this only works on 64 bit, on 32 bits the alignment and reference size is fixed):
- boolean compressedOops = false;
- int objectAlignment = 8;
- boolean isHotspot = false;
- try {
- final Class<?> beanClazz = Class.forName(HOTSPOT_BEAN_CLASS);
- // we use reflection for this, because the management factory is not part
- // of Java 8's compact profile:
- final Object hotSpotBean =
- Class.forName(MANAGEMENT_FACTORY_CLASS)
- .getMethod("getPlatformMXBean", Class.class)
- .invoke(null, beanClazz);
- if (hotSpotBean != null) {
- isHotspot = true;
- final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class);
- try {
- final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "UseCompressedOops");
- compressedOops =
- Boolean.parseBoolean(
- vmOption.getClass().getMethod("getValue").invoke(vmOption).toString());
- } catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {
- isHotspot = false;
- }
- try {
- final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "ObjectAlignmentInBytes");
- objectAlignment =
- Integer.parseInt(
- vmOption.getClass().getMethod("getValue").invoke(vmOption).toString());
- } catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {
- isHotspot = false;
- }
- }
- } catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {
- isHotspot = false;
- final Logger log = Logger.getLogger(RamUsageEstimator.class.getName());
- final Module module = RamUsageEstimator.class.getModule();
- final ModuleLayer layer = module.getLayer();
- // classpath / unnamed module has no layer, so we need to check:
- if (layer != null
- && layer.findModule("jdk.management").map(module::canRead).orElse(false) == false) {
- log.warning(
- "Lucene cannot correctly calculate object sizes on 64bit JVMs, unless the 'jdk.management' Java module "
- + "is readable [please add 'jdk.management' to modular application either by command line or its module descriptor]");
- } else {
- log.warning(
- "Lucene cannot correctly calculate object sizes on 64bit JVMs that are not based on Hotspot or a compatible implementation.");
- }
- }
- JVM_IS_HOTSPOT_64BIT = isHotspot;
- COMPRESSED_REFS_ENABLED = compressedOops;
- NUM_BYTES_OBJECT_ALIGNMENT = objectAlignment;
+ JVM_IS_HOTSPOT_64BIT = true;
+ COMPRESSED_REFS_ENABLED =
+ HotspotVMOptions.get("UseCompressedOops").map(Boolean::valueOf).orElse(false);
+ NUM_BYTES_OBJECT_ALIGNMENT =
+ HotspotVMOptions.get("ObjectAlignmentInBytes").map(Integer::valueOf).orElse(8);
// reference size is 4, if we have compressed oops:
NUM_BYTES_OBJECT_REF = COMPRESSED_REFS_ENABLED ? 4 : 8;
// "best guess" based on reference size:
diff --git a/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
index 5b382c4c7c2..d4e8a50ef8f 100644
--- a/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
+++ b/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
@@ -77,41 +77,9 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {
VectorizationProvider.TESTS_FORCE_INTEGER_VECTORS || (isAMD64withoutAVX2 == false);
}
- private static final String MANAGEMENT_FACTORY_CLASS = "java.lang.management.ManagementFactory";
- private static final String HOTSPOT_BEAN_CLASS = "com.sun.management.HotSpotDiagnosticMXBean";
-
- // best effort to see if FMA is fast (this is architecture-independent option)
- private static boolean hasFastFMA() {
- // on ARM cpus, FMA works fine but is a slight slowdown: don't use it.
- if (Constants.OS_ARCH.equals("amd64") == false) {
- return false;
- }
- try {
- final Class<?> beanClazz = Class.forName(HOTSPOT_BEAN_CLASS);
- // we use reflection for this, because the management factory is not part
- // of Java 8's compact profile:
- final Object hotSpotBean =
- Class.forName(MANAGEMENT_FACTORY_CLASS)
- .getMethod("getPlatformMXBean", Class.class)
- .invoke(null, beanClazz);
- if (hotSpotBean != null) {
- final var getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class);
- final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "UseFMA");
- return Boolean.parseBoolean(
- vmOption.getClass().getMethod("getValue").invoke(vmOption).toString());
- }
- return false;
- } catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {
- return false;
- }
- }
-
- // true if we know FMA is supported, to deliver less error
- static final boolean HAS_FAST_FMA = hasFastFMA();
-
// the way FMA should work! if available use it, otherwise fall back to mul/add
private static FloatVector fma(FloatVector a, FloatVector b, FloatVector c) {
- if (HAS_FAST_FMA) {
+ if (Constants.HAS_FAST_FMA) {
return a.fma(b, c);
} else {
return a.mul(b).add(c);
diff --git a/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java b/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java
index fc303a687a0..ffd18df1a27 100644
--- a/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java
+++ b/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java
@@ -21,6 +21,7 @@ import java.security.PrivilegedAction;
import java.util.Locale;
import java.util.logging.Logger;
import jdk.incubator.vector.FloatVector;
+import org.apache.lucene.util.Constants;
import org.apache.lucene.util.SuppressForbidden;
/** A vectorization provider that leverages the Panama Vector API. */
@@ -62,7 +63,7 @@ final class PanamaVectorizationProvider extends VectorizationProvider {
Locale.ENGLISH,
"Java vector incubator API enabled; uses preferredBitSize=%d%s%s",
PanamaVectorUtilSupport.VECTOR_BITSIZE,
- PanamaVectorUtilSupport.HAS_FAST_FMA ? "; FMA enabled" : "",
+ Constants.HAS_FAST_FMA ? "; FMA enabled" : "",
PanamaVectorUtilSupport.HAS_FAST_INTEGER_VECTORS
? ""
: "; floating-point vectors only"));