You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@accumulo.apache.org by GitBox <gi...@apache.org> on 2020/11/04 21:06:55 UTC
[GitHub] [accumulo] ctubbsii commented on a change in pull request #1715: Deprecation of VFS Class Loader items, new Context class loader management
ctubbsii commented on a change in pull request #1715:
URL: https://github.com/apache/accumulo/pull/1715#discussion_r517533335
##########
File path: core/src/main/java/org/apache/accumulo/core/classloader/ContextClassLoaders.java
##########
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ContextClassLoaders {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ContextClassLoaders.class);
+
+ public static final String CONTEXT_CLASS_LOADER_FACTORY = "general.context.class.loader.factory";
+
+ private static ContextClassLoaderFactory FACTORY;
+ private static Supplier<Map<String,String>> CONF;
+
+ /**
+ * Initialize the ContextClassLoaderFactory
+ *
+ * @param conf
+ * AccumuloConfiguration object
+ */
+ @SuppressWarnings("unchecked")
Review comment:
Can we move this warnings suppression closer to the objects that it applies to, if we must have it?
##########
File path: core/src/main/java/org/apache/accumulo/core/classloader/ContextClassLoaders.java
##########
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ContextClassLoaders {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ContextClassLoaders.class);
+
+ public static final String CONTEXT_CLASS_LOADER_FACTORY = "general.context.class.loader.factory";
+
+ private static ContextClassLoaderFactory FACTORY;
+ private static Supplier<Map<String,String>> CONF;
+
+ /**
+ * Initialize the ContextClassLoaderFactory
+ *
+ * @param conf
+ * AccumuloConfiguration object
+ */
+ @SuppressWarnings("unchecked")
+ public static void initialize(Supplier<Map<String,String>> conf) throws Exception {
+ if (null == CONF) {
+ CONF = conf;
+ LOG.info("Creating ContextClassLoaderFactory");
+ var factoryName = CONF.get().get(Property.GENERAL_CONTEXT_CLASSLOADER_FACTORY.toString());
Review comment:
I think this should call `.getKey()` instead of `.toString()`
##########
File path: core/src/main/java/org/apache/accumulo/core/classloader/ClassLoaderUtil.java
##########
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+
+public class ClassLoaderUtil {
+
+ public static <U> Class<? extends U> loadClass(String contextName, String className,
+ Class<U> extension) throws ClassNotFoundException {
+ if (contextName != null && !contextName.equals(""))
+ return ContextClassLoaders.getClassLoader(contextName).loadClass(className)
+ .asSubclass(extension);
+ else
+ return AccumuloVFSClassLoader.loadClass(className, extension);
+
+ }
Review comment:
Is there not an appropriate existing utility class where this single static method can live? I think there's similar methods in the ConfigurationTypeHelper class.
##########
File path: core/src/main/java/org/apache/accumulo/core/classloader/ClassLoaderUtil.java
##########
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+
+public class ClassLoaderUtil {
+
+ public static <U> Class<? extends U> loadClass(String contextName, String className,
+ Class<U> extension) throws ClassNotFoundException {
+ if (contextName != null && !contextName.equals(""))
Review comment:
```suggestion
if (contextName != null && !contextName.isEmpty())
```
##########
File path: core/src/main/java/org/apache/accumulo/core/conf/Property.java
##########
@@ -196,6 +198,9 @@
+ "Additionally, this property no longer does property interpolation of environment "
+ "variables, such as '$ACCUMULO_HOME'. Use commons-configuration syntax,"
+ "'${env:ACCUMULO_HOME}' instead."),
+ GENERAL_CONTEXT_CLASSLOADER_FACTORY(ContextClassLoaders.CONTEXT_CLASS_LOADER_FACTORY,
+ LegacyVFSContextClassLoaderFactory.class.getName(), PropertyType.STRING,
+ "Name of classloader factory to be used to create classloaders for named contexts."),
Review comment:
We should inline this property's key. I don't think there's a reason to have that string located in a separate constant outside this class. That was only done for some of the VFS stuff that existed in the start module. But, this constant is just in a different location in the core module.
##########
File path: core/src/test/java/org/apache/accumulo/core/classloader/URLClassLoaderFactory.java
##########
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+
+import org.apache.accumulo.core.client.PluginEnvironment.Configuration;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class URLClassLoaderFactory implements ContextClassLoaderFactory {
+
+ private static final String COMMA = ",";
Review comment:
Should we use semicolon as the delimiter, because so this behaves more like a `CLASSPATH`?
##########
File path: core/src/test/java/org/apache/accumulo/core/classloader/URLClassLoaderFactory.java
##########
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+
+import org.apache.accumulo.core.client.PluginEnvironment.Configuration;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class URLClassLoaderFactory implements ContextClassLoaderFactory {
+
+ private static final String COMMA = ",";
+ private static final Logger LOG = LoggerFactory.getLogger(URLClassLoaderFactory.class);
+
+ @Override
+ public void initialize(Configuration contextProperties) throws Exception {}
+
+ @Override
+ public ClassLoader getClassLoader(String contextName) throws IllegalArgumentException {
+ // The context name is the classpath.
+ var parts = contextName.split(COMMA);
+ var urls = new ArrayList<URL>();
+ for (String p : parts) {
+ try {
+ urls.add(new URL(p));
+ } catch (MalformedURLException e) {
+ LOG.error("Error creating URL from classpath segment: " + p);
+ throw new RuntimeException("Error creating URL from classpath segment: " + p, e);
Review comment:
It would be good to use more specific RTEs instead of the generic one, whenever a more appropriate one is available. Here, IllegalArgumentException would be appropriate.
##########
File path: core/src/test/java/org/apache/accumulo/core/classloader/URLClassLoaderFactory.java
##########
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+
+import org.apache.accumulo.core.client.PluginEnvironment.Configuration;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class URLClassLoaderFactory implements ContextClassLoaderFactory {
+
+ private static final String COMMA = ",";
+ private static final Logger LOG = LoggerFactory.getLogger(URLClassLoaderFactory.class);
+
+ @Override
+ public void initialize(Configuration contextProperties) throws Exception {}
+
+ @Override
+ public ClassLoader getClassLoader(String contextName) throws IllegalArgumentException {
+ // The context name is the classpath.
+ var parts = contextName.split(COMMA);
+ var urls = new ArrayList<URL>();
+ for (String p : parts) {
+ try {
+ urls.add(new URL(p));
+ } catch (MalformedURLException e) {
Review comment:
This can probably be done a bit more succinctly with streams, something like:
```java
var urls = Stream.of(contextName.split(DELIM)).map( p -> toUrl(p) ).collect(Collectors.toList());
```
##########
File path: core/src/main/java/org/apache/accumulo/core/classloader/ContextClassLoaders.java
##########
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ContextClassLoaders {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ContextClassLoaders.class);
+
+ public static final String CONTEXT_CLASS_LOADER_FACTORY = "general.context.class.loader.factory";
+
+ private static ContextClassLoaderFactory FACTORY;
Review comment:
Since initialize isn't synchronized, do we want to do something here to prevent this from being changed (by a competing thread) once set?
##########
File path: core/src/main/java/org/apache/accumulo/core/classloader/LegacyVFSContextClassLoaderFactory.java
##########
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.function.Supplier;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Deprecated(since = "2.1.0", forRemoval = true)
+public class LegacyVFSContextClassLoaderFactory implements ContextClassLoaderFactory {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LegacyVFSContextClassLoaderFactory.class);
+
+ public void initialize(Supplier<Map<String,String>> contextProperties) {
+ try {
+ AccumuloVFSClassLoader.getContextManager()
+ .setContextConfig(new ContextManager.DefaultContextsConfig() {
+ @Override
+ public Map<String,String> getVfsContextClasspathProperties() {
+ return contextProperties.get();
+ }
+ });
+ LOG.debug("ContextManager configuration set");
+ new Timer("LegacyVFSContextClassLoaderFactory-cleanup", true)
+ .scheduleAtFixedRate(new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("LegacyVFSContextClassLoaderFactory-cleanup thread, properties: {}",
+ contextProperties.get());
+ }
+ Set<String> configuredContexts = new HashSet<>();
+ contextProperties.get().keySet().forEach(k -> {
+ if (k.startsWith(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.toString())) {
+ configuredContexts.add(
+ k.substring(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.toString().length()));
+ }
+ });
+ LOG.trace("LegacyVFSContextClassLoaderFactory-cleanup thread, contexts in use: {}",
+ configuredContexts);
+ AccumuloVFSClassLoader.getContextManager().removeUnusedContexts(configuredContexts);
+ } catch (IOException e) {
+ LOG.warn("{}", e.getMessage(), e);
+ }
+ }
+ }, 60000, 60000);
+ LOG.debug("Context cleanup timer started at 60s intervals");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
Review comment:
UncheckedIOException should be used to wrap IOException as a runtime exception.
##########
File path: core/src/main/java/org/apache/accumulo/core/classloader/ContextClassLoaders.java
##########
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ContextClassLoaders {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ContextClassLoaders.class);
+
+ public static final String CONTEXT_CLASS_LOADER_FACTORY = "general.context.class.loader.factory";
+
+ private static ContextClassLoaderFactory FACTORY;
+ private static Supplier<Map<String,String>> CONF;
+
+ /**
+ * Initialize the ContextClassLoaderFactory
+ *
+ * @param conf
+ * AccumuloConfiguration object
+ */
+ @SuppressWarnings("unchecked")
+ public static void initialize(Supplier<Map<String,String>> conf) throws Exception {
+ if (null == CONF) {
+ CONF = conf;
+ LOG.info("Creating ContextClassLoaderFactory");
+ var factoryName = CONF.get().get(Property.GENERAL_CONTEXT_CLASSLOADER_FACTORY.toString());
+ if (null == factoryName || factoryName.isBlank()) {
+ LOG.info("No ClassLoaderFactory specified");
+ return;
+ }
+ try {
+ var factoryClass = Class.forName(factoryName);
Review comment:
You used `isBlank` above to check this instead of `isEmpty`, which implies that the string is not already stripped of whitespace. So, that implies `factoryName` might need to be stripped of surrounding whitespace. If it's not possible for it to have surrounding whitespace, then `isEmpty` might be more appropriate above than `isBlank`.
##########
File path: core/src/main/java/org/apache/accumulo/core/classloader/LegacyVFSContextClassLoaderFactory.java
##########
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.function.Supplier;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Deprecated(since = "2.1.0", forRemoval = true)
+public class LegacyVFSContextClassLoaderFactory implements ContextClassLoaderFactory {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LegacyVFSContextClassLoaderFactory.class);
+
+ public void initialize(Supplier<Map<String,String>> contextProperties) {
+ try {
+ AccumuloVFSClassLoader.getContextManager()
+ .setContextConfig(new ContextManager.DefaultContextsConfig() {
+ @Override
+ public Map<String,String> getVfsContextClasspathProperties() {
+ return contextProperties.get();
+ }
+ });
+ LOG.debug("ContextManager configuration set");
+ new Timer("LegacyVFSContextClassLoaderFactory-cleanup", true)
+ .scheduleAtFixedRate(new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("LegacyVFSContextClassLoaderFactory-cleanup thread, properties: {}",
+ contextProperties.get());
+ }
+ Set<String> configuredContexts = new HashSet<>();
+ contextProperties.get().keySet().forEach(k -> {
+ if (k.startsWith(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.toString())) {
Review comment:
Should use `.getKey()` instead of `.toString()`.
##########
File path: core/src/main/java/org/apache/accumulo/core/classloader/LegacyVFSContextClassLoaderFactory.java
##########
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.function.Supplier;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Deprecated(since = "2.1.0", forRemoval = true)
+public class LegacyVFSContextClassLoaderFactory implements ContextClassLoaderFactory {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LegacyVFSContextClassLoaderFactory.class);
+
+ public void initialize(Supplier<Map<String,String>> contextProperties) {
+ try {
+ AccumuloVFSClassLoader.getContextManager()
+ .setContextConfig(new ContextManager.DefaultContextsConfig() {
+ @Override
+ public Map<String,String> getVfsContextClasspathProperties() {
+ return contextProperties.get();
+ }
+ });
+ LOG.debug("ContextManager configuration set");
+ new Timer("LegacyVFSContextClassLoaderFactory-cleanup", true)
+ .scheduleAtFixedRate(new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("LegacyVFSContextClassLoaderFactory-cleanup thread, properties: {}",
+ contextProperties.get());
+ }
+ Set<String> configuredContexts = new HashSet<>();
+ contextProperties.get().keySet().forEach(k -> {
+ if (k.startsWith(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.toString())) {
+ configuredContexts.add(
+ k.substring(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.toString().length()));
+ }
+ });
+ LOG.trace("LegacyVFSContextClassLoaderFactory-cleanup thread, contexts in use: {}",
+ configuredContexts);
+ AccumuloVFSClassLoader.getContextManager().removeUnusedContexts(configuredContexts);
+ } catch (IOException e) {
+ LOG.warn("{}", e.getMessage(), e);
+ }
+ }
+ }, 60000, 60000);
+ LOG.debug("Context cleanup timer started at 60s intervals");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ @Override
+ public ClassLoader getClassLoader(String contextName) throws IllegalArgumentException {
+ try {
+ return AccumuloVFSClassLoader.getContextManager().getClassLoader(contextName);
+ } catch (IOException e) {
+ throw new RuntimeException("Error getting context class loader for context: " + contextName,
Review comment:
UncheckedIOException
##########
File path: core/src/main/java/org/apache/accumulo/core/classloader/ContextClassLoaders.java
##########
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ContextClassLoaders {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ContextClassLoaders.class);
+
+ public static final String CONTEXT_CLASS_LOADER_FACTORY = "general.context.class.loader.factory";
+
+ private static ContextClassLoaderFactory FACTORY;
+ private static Supplier<Map<String,String>> CONF;
+
+ /**
+ * Initialize the ContextClassLoaderFactory
+ *
+ * @param conf
+ * AccumuloConfiguration object
+ */
+ @SuppressWarnings("unchecked")
+ public static void initialize(Supplier<Map<String,String>> conf) throws Exception {
+ if (null == CONF) {
+ CONF = conf;
+ LOG.info("Creating ContextClassLoaderFactory");
+ var factoryName = CONF.get().get(Property.GENERAL_CONTEXT_CLASSLOADER_FACTORY.toString());
+ if (null == factoryName || factoryName.isBlank()) {
+ LOG.info("No ClassLoaderFactory specified");
+ return;
+ }
+ try {
+ var factoryClass = Class.forName(factoryName);
+ if (ContextClassLoaderFactory.class.isAssignableFrom(factoryClass)) {
+ LOG.info("Creating ContextClassLoaderFactory: {}", factoryName);
+ FACTORY = ((Class<? extends ContextClassLoaderFactory>) factoryClass)
+ .getDeclaredConstructor().newInstance();
+ FACTORY.initialize(new Supplier<Map<String,String>>() {
+ @Override
+ public Map<String,String> get() {
+ return CONF.get();
+ }
+ });
+ } else {
+ throw new RuntimeException(factoryName + " does not implement ContextClassLoaderFactory");
+ }
+ } catch (ClassNotFoundException | InstantiationException | IllegalAccessException
+ | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
+ | SecurityException e) {
+ LOG.error(
+ "Unable to load and initialize class: {}. Ensure that the jar containing the ContextClassLoaderFactory is on the classpath",
+ factoryName);
+ throw e;
+ }
+ } else {
+ LOG.debug("ContextClassLoaderFactory already initialized.");
+ }
+ }
+
+ /**
+ * Return the ClassLoader for the given contextName
+ *
+ * @param contextName
+ * name
+ * @return ClassLoader for contextName, do not cache this
+ * @throws RuntimeException
+ * if contextName not configured
+ */
+ public static ClassLoader getClassLoader(String contextName) {
+ try {
+ // Cannot cache the ClassLoader result as it
+ // may change when the ClassLoader reloads
+ return FACTORY.getClassLoader(contextName);
+ } catch (IllegalArgumentException e) {
+ LOG.error("ContextClassLoaderFactory is not configured for context: {}", contextName);
+ throw new RuntimeException(
+ "ContextClassLoaderFactory is not configured for context: " + contextName);
Review comment:
You're catching an RTE to log and then throw a different RTE. This adds redundant logs and makes it harder to get a meaningful stack trace. It'd be better to just let the IAE fall through.
##########
File path: core/src/main/java/org/apache/accumulo/core/classloader/ContextClassLoaders.java
##########
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ContextClassLoaders {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ContextClassLoaders.class);
+
+ public static final String CONTEXT_CLASS_LOADER_FACTORY = "general.context.class.loader.factory";
+
+ private static ContextClassLoaderFactory FACTORY;
+ private static Supplier<Map<String,String>> CONF;
+
+ /**
+ * Initialize the ContextClassLoaderFactory
+ *
+ * @param conf
+ * AccumuloConfiguration object
+ */
+ @SuppressWarnings("unchecked")
+ public static void initialize(Supplier<Map<String,String>> conf) throws Exception {
+ if (null == CONF) {
+ CONF = conf;
+ LOG.info("Creating ContextClassLoaderFactory");
+ var factoryName = CONF.get().get(Property.GENERAL_CONTEXT_CLASSLOADER_FACTORY.toString());
+ if (null == factoryName || factoryName.isBlank()) {
+ LOG.info("No ClassLoaderFactory specified");
+ return;
+ }
+ try {
+ var factoryClass = Class.forName(factoryName);
+ if (ContextClassLoaderFactory.class.isAssignableFrom(factoryClass)) {
+ LOG.info("Creating ContextClassLoaderFactory: {}", factoryName);
+ FACTORY = ((Class<? extends ContextClassLoaderFactory>) factoryClass)
+ .getDeclaredConstructor().newInstance();
+ FACTORY.initialize(new Supplier<Map<String,String>>() {
+ @Override
+ public Map<String,String> get() {
+ return CONF.get();
+ }
+ });
Review comment:
```suggestion
FACTORY.initialize(() -> CONF.get());
```
##########
File path: core/src/main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java
##########
@@ -49,9 +50,17 @@ public ConfigurationCopy(Map<String,String> config) {
* configuration property iterable to use for copying
*/
public ConfigurationCopy(Iterable<Entry<String,String>> config) {
- for (Entry<String,String> entry : config) {
- copy.put(entry.getKey(), entry.getValue());
- }
+ this(config.iterator());
+ }
+
+ /**
+ * Creates a new configuration.
+ *
+ * @param config
+ * configuration property iterator to use for copying
+ */
+ public ConfigurationCopy(Iterator<Entry<String,String>> config) {
+ config.forEachRemaining(e -> copy.put(e.getKey(), e.getValue()));
Review comment:
It looks like this constructor was added, but not used.
##########
File path: core/src/main/java/org/apache/accumulo/core/classloader/ContextClassLoaders.java
##########
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.lang.reflect.InvocationTargetException;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.apache.accumulo.core.util.ConfigurationImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ContextClassLoaders {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ContextClassLoaders.class);
+
+ public static final String CONTEXT_CLASS_LOADER_FACTORY = "general.context.class.loader.factory";
+
+ private static ContextClassLoaderFactory FACTORY;
+ private static AccumuloConfiguration CONF;
+
+ /**
+ * Initialize the ContextClassLoaderFactory
+ *
+ * @param conf
+ * AccumuloConfiguration object
+ */
+ @SuppressWarnings("unchecked")
+ public static void initialize(AccumuloConfiguration conf) throws Exception {
+ if (null == CONF) {
+ CONF = conf;
+ LOG.info("Creating ContextClassLoaderFactory");
+ var factoryName = CONF.get(Property.GENERAL_CONTEXT_CLASSLOADER_FACTORY.toString());
+ if (null == factoryName || factoryName.isBlank()) {
+ LOG.info("No ClassLoaderFactory specified");
+ return;
+ }
+ try {
+ var factoryClass = Class.forName(factoryName);
+ if (ContextClassLoaderFactory.class.isAssignableFrom(factoryClass)) {
+ LOG.info("Creating ContextClassLoaderFactory: {}", factoryName);
+ FACTORY = ((Class<? extends ContextClassLoaderFactory>) factoryClass)
+ .getDeclaredConstructor().newInstance();
+ FACTORY.initialize(new ConfigurationImpl(CONF));
+ } else {
+ throw new RuntimeException(factoryName + " does not implement ContextClassLoaderFactory");
+ }
+ } catch (ClassNotFoundException | InstantiationException | IllegalAccessException
+ | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
+ | SecurityException e) {
+ LOG.error(
+ "Unable to load and initialize class: {}. Ensure that the jar containing the ContextClassLoaderFactory is on the classpath",
+ factoryName);
+ throw e;
+ }
+ } else {
+ LOG.debug("ContextClassLoaderFactory already initialized.");
+ }
+ }
+
+ /**
+ * Return the ClassLoader for the given contextName
+ *
+ * @param contextName
+ * name
+ * @return ClassLoader for contextName, do not cache this
+ * @throws RuntimeException
+ * if contextName not configured
+ */
+ public static ClassLoader getClassLoader(String contextName) {
Review comment:
Is this method even necessary? Could just have a getter for the FACTORY. I'm not sure this static method adds much in terms of convenience.
##########
File path: server/manager/src/main/java/org/apache/accumulo/master/Master.java
##########
@@ -378,14 +377,12 @@ public static void main(String[] args) throws Exception {
Property.MASTER_TABLET_BALANCER, TabletBalancer.class, new DefaultLoadBalancer());
this.tabletBalancer.init(context);
- AccumuloVFSClassLoader.getContextManager()
- .setContextConfig(new ContextManager.DefaultContextsConfig() {
- @Override
- public Map<String,String> getVfsContextClasspathProperties() {
- return getConfiguration()
- .getAllPropertiesWithPrefix(Property.VFS_CONTEXT_CLASSPATH_PROPERTY);
- }
- });
+ try {
+ ContextClassLoaders.initialize(aconf);
+ } catch (Exception e1) {
+ log.error("Error configuring ContextClassLoaderFactory", e1);
+ throw new RuntimeException("Error configuring ContextClassLoaderFactory", e1);
+ }
Review comment:
Maybe only catch checked exceptions here. Anything falling through that isn't a checked exception is already an RTE and we don't need to rewrap as a new RTE.
##########
File path: core/src/test/java/org/apache/accumulo/core/classloader/URLClassLoaderFactory.java
##########
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.classloader;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+
+import org.apache.accumulo.core.client.PluginEnvironment.Configuration;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class URLClassLoaderFactory implements ContextClassLoaderFactory {
+
+ private static final String COMMA = ",";
+ private static final Logger LOG = LoggerFactory.getLogger(URLClassLoaderFactory.class);
+
+ @Override
+ public void initialize(Configuration contextProperties) throws Exception {}
+
+ @Override
+ public ClassLoader getClassLoader(String contextName) throws IllegalArgumentException {
+ // The context name is the classpath.
+ var parts = contextName.split(COMMA);
+ var urls = new ArrayList<URL>();
+ for (String p : parts) {
+ try {
+ urls.add(new URL(p));
+ } catch (MalformedURLException e) {
+ LOG.error("Error creating URL from classpath segment: " + p);
Review comment:
Should log or throw, but not both. This is how we get duplicate log messages in the log files. The caller can choose to log as part of its handling mechanism, if you choose to throw.
##########
File path: start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
##########
@@ -358,48 +360,24 @@ public static void printClassPath(Printer out, boolean debug) {
continue;
}
- String classLoaderDescription;
- switch (level) {
- case 1:
- classLoaderDescription =
- level + ": Java System Classloader (loads Java system resources)";
- break;
- case 2:
- classLoaderDescription =
- level + ": Java Classloader (loads everything defined by java classpath)";
- break;
- case 3:
- classLoaderDescription =
- level + ": Accumulo Classloader (loads everything defined by general.classpaths)";
- break;
- case 4:
- classLoaderDescription = level + ": Accumulo Dynamic Classloader "
- + "(loads everything defined by general.dynamic.classpaths)";
- break;
- default:
- classLoaderDescription = level + ": Mystery Classloader ("
- + "someone probably added a classloader and didn't update the switch statement in "
- + AccumuloVFSClassLoader.class.getName() + ")";
- break;
- }
-
boolean sawFirst = false;
+ String classLoaderDescription = "Level: " + level + ", Name: " + classLoader.getName()
+ + ", class: " + classLoader.getClass().getName();
if (classLoader.getClass().getName().startsWith("jdk.internal")) {
if (debug) {
- out.print("Level " + classLoaderDescription + " " + classLoader.getClass().getName()
- + " configuration not inspectable.\n");
+ out.print(classLoaderDescription + ": configuration not inspectable.\n");
Review comment:
I think there might be an IT that checks the printed strings... not sure. It's fine if it changes, but the IT might need to be updated.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org