You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2023/08/29 04:23:31 UTC
[camel] branch main updated: Java dsl eager (#11222)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new b53b865b113 Java dsl eager (#11222)
b53b865b113 is described below
commit b53b865b1132e4aa59903d9d2a2be12555964e67
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Tue Aug 29 06:23:24 2023 +0200
Java dsl eager (#11222)
* CAMEL-19795: camel-joor-dsl: Make it possible to eager preParse all .java files together in the same compilation unit.
---
.../camel/spi/ExtendedRoutesBuilderLoader.java | 15 +++
.../camel/impl/engine/DefaultRoutesLoader.java | 54 ++++++++---
.../org/apache/camel/main/RoutesConfigurer.java | 73 +++++++++++++--
.../camel/dsl/java/joor/JavaJoorClassLoader.java | 8 +-
.../dsl/java/joor/JavaRoutesBuilderLoader.java | 101 +++++++++++++++------
.../yaml/IntegrationLoaderDependenciesTest.groovy | 2 +-
6 files changed, 196 insertions(+), 57 deletions(-)
diff --git a/core/camel-api/src/main/java/org/apache/camel/spi/ExtendedRoutesBuilderLoader.java b/core/camel-api/src/main/java/org/apache/camel/spi/ExtendedRoutesBuilderLoader.java
index 8528775e58e..06ca1520206 100644
--- a/core/camel-api/src/main/java/org/apache/camel/spi/ExtendedRoutesBuilderLoader.java
+++ b/core/camel-api/src/main/java/org/apache/camel/spi/ExtendedRoutesBuilderLoader.java
@@ -26,6 +26,21 @@ import org.apache.camel.RoutesBuilder;
*/
public interface ExtendedRoutesBuilderLoader extends RoutesBuilderLoader {
+ /**
+ * Pre-parses the {@link RoutesBuilder} from multiple {@link Resource}s.
+ *
+ * This is used during bootstrap, to eager detect configurations from route DSL resources which makes it possible to
+ * specify configurations that affect the bootstrap, such as by camel-jbang and camel-yaml-dsl.
+ *
+ * @param resources the resources to be pre parsed.
+ */
+ default void preParseRoutes(Collection<Resource> resources) throws Exception {
+ // by default parse one-by-one
+ for (Resource resource : resources) {
+ preParseRoute(resource);
+ }
+ }
+
/**
* Loads {@link RoutesBuilder} from multiple {@link Resource}s.
*
diff --git a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRoutesLoader.java b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRoutesLoader.java
index ecf43bfa44b..37196f50993 100644
--- a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRoutesLoader.java
+++ b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRoutesLoader.java
@@ -92,24 +92,22 @@ public class DefaultRoutesLoader extends ServiceSupport implements RoutesLoader,
public Collection<RoutesBuilder> findRoutesBuilders(Collection<Resource> resources, boolean optional) throws Exception {
List<RoutesBuilder> answer = new ArrayList<>(resources.size());
- // first we need to parse for modeline to gather all the configurations
- if (camelContext.isModeline()) {
- ModelineFactory factory = PluginHelper.getModelineFactory(camelContext);
- for (Resource resource : resources) {
- try (RoutesBuilderLoader loader = resolveRoutesBuilderLoader(resource, optional)) {
- if (loader != null) {
- // gather resources for modeline
- factory.parseModeline(resource);
- // pre-parse before loading
- loader.preParseRoute(resource);
- }
- }
+ // sort groups so java is first
+ List<Resource> sort = new ArrayList<>(resources);
+ sort.sort((o1, o2) -> {
+ String ext1 = FileUtil.onlyExt(o1.getLocation(), false);
+ String ext2 = FileUtil.onlyExt(o2.getLocation(), false);
+ if ("java".equals(ext1)) {
+ return -1;
+ } else if ("java".equals(ext2)) {
+ return 1;
}
- }
+ return 0;
+ });
- // now group resources by loader
+ // group resources by loader (java, xml, yaml in their own group)
Map<RoutesBuilderLoader, List<Resource>> groups = new LinkedHashMap<>();
- for (Resource resource : resources) {
+ for (Resource resource : sort) {
RoutesBuilderLoader loader = resolveRoutesBuilderLoader(resource, optional);
if (loader != null) {
List<Resource> list = groups.getOrDefault(loader, new ArrayList<>());
@@ -118,6 +116,32 @@ public class DefaultRoutesLoader extends ServiceSupport implements RoutesLoader,
}
}
+ // first we need to parse for modeline to gather all the configurations
+ if (camelContext.isModeline()) {
+ ModelineFactory factory = PluginHelper.getModelineFactory(camelContext);
+ for (Map.Entry<RoutesBuilderLoader, List<Resource>> entry : groups.entrySet()) {
+ // parse modelines for all resources
+ for (Resource resource : entry.getValue()) {
+ factory.parseModeline(resource);
+ }
+ }
+ }
+
+ // then pre-parse routes
+ for (Map.Entry<RoutesBuilderLoader, List<Resource>> entry : groups.entrySet()) {
+ RoutesBuilderLoader loader = entry.getKey();
+ if (loader instanceof ExtendedRoutesBuilderLoader) {
+ // extended loader can load all resources ine one unit
+ ExtendedRoutesBuilderLoader extLoader = (ExtendedRoutesBuilderLoader) loader;
+ // pre-parse before loading
+ extLoader.preParseRoutes(entry.getValue());
+ } else {
+ for (Resource resource : entry.getValue()) {
+ loader.preParseRoute(resource);
+ }
+ }
+ }
+
// now load all the same resources for each loader
for (Map.Entry<RoutesBuilderLoader, List<Resource>> entry : groups.entrySet()) {
RoutesBuilderLoader loader = entry.getKey();
diff --git a/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java b/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java
index 36558c59395..5b9a7fd50e7 100644
--- a/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java
+++ b/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java
@@ -18,7 +18,9 @@ package org.apache.camel.main;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.StringJoiner;
@@ -27,11 +29,15 @@ import org.apache.camel.RouteConfigurationsBuilder;
import org.apache.camel.RoutesBuilder;
import org.apache.camel.RuntimeCamelException;
import org.apache.camel.spi.CamelBeanPostProcessor;
+import org.apache.camel.spi.ExtendedRoutesBuilderLoader;
import org.apache.camel.spi.ModelineFactory;
import org.apache.camel.spi.Resource;
+import org.apache.camel.spi.RoutesBuilderLoader;
import org.apache.camel.spi.RoutesLoader;
import org.apache.camel.support.OrderedComparator;
import org.apache.camel.support.PluginHelper;
+import org.apache.camel.util.FileUtil;
+import org.apache.camel.util.ObjectHelper;
import org.apache.camel.util.StopWatch;
import org.apache.camel.util.TimeUtils;
import org.slf4j.Logger;
@@ -302,20 +308,73 @@ public class RoutesConfigurer {
protected void doConfigureModeline(CamelContext camelContext, Collection<Resource> resources, boolean optional)
throws Exception {
- RoutesLoader loader = PluginHelper.getRoutesLoader(camelContext);
+
+ // sort groups so java is first
+ List<Resource> sort = new ArrayList<>(resources);
+ sort.sort((o1, o2) -> {
+ String ext1 = FileUtil.onlyExt(o1.getLocation(), false);
+ String ext2 = FileUtil.onlyExt(o2.getLocation(), false);
+ if ("java".equals(ext1)) {
+ return -1;
+ } else if ("java".equals(ext2)) {
+ return 1;
+ }
+ return 0;
+ });
+
+ // group resources by loader (java, xml, yaml in their own group)
+ Map<RoutesBuilderLoader, List<Resource>> groups = new LinkedHashMap<>();
+ for (Resource resource : sort) {
+ RoutesBuilderLoader loader = resolveRoutesBuilderLoader(camelContext, resource, optional);
+ if (loader != null) {
+ List<Resource> list = groups.getOrDefault(loader, new ArrayList<>());
+ list.add(resource);
+ groups.put(loader, list);
+ }
+ }
if (camelContext.isModeline()) {
+ // parse modelines for all resources
ModelineFactory factory = PluginHelper.getModelineFactory(camelContext);
- for (Resource resource : resources) {
- LOG.debug("Parsing modeline: {}", resource);
- factory.parseModeline(resource);
+ for (Map.Entry<RoutesBuilderLoader, List<Resource>> entry : groups.entrySet()) {
+ for (Resource resource : entry.getValue()) {
+ factory.parseModeline(resource);
+ }
}
}
+
// the resource may also have additional configurations which we need to detect via pre-parsing
- for (Resource resource : resources) {
- LOG.debug("Pre-parsing: {}", resource);
- loader.preParseRoute(resource, optional);
+ for (Map.Entry<RoutesBuilderLoader, List<Resource>> entry : groups.entrySet()) {
+ RoutesBuilderLoader loader = entry.getKey();
+ if (loader instanceof ExtendedRoutesBuilderLoader) {
+ // extended loader can pre-parse all resources ine one unit
+ ExtendedRoutesBuilderLoader extLoader = (ExtendedRoutesBuilderLoader) loader;
+ extLoader.preParseRoutes(entry.getValue());
+ } else {
+ for (Resource resource : entry.getValue()) {
+ loader.preParseRoute(resource);
+ }
+ }
+ }
+ }
+
+ protected RoutesBuilderLoader resolveRoutesBuilderLoader(CamelContext camelContext, Resource resource,
+ boolean optional) throws Exception {
+ // the loader to use is derived from the file extension
+ final String extension = FileUtil.onlyExt(resource.getLocation(), false);
+
+ if (ObjectHelper.isEmpty(extension)) {
+ throw new IllegalArgumentException(
+ "Unable to determine file extension for resource: " + resource.getLocation());
+ }
+
+ RoutesLoader loader = PluginHelper.getRoutesLoader(camelContext);
+ RoutesBuilderLoader answer = loader.getRoutesLoader(extension);
+ if (!optional && answer == null) {
+ throw new IllegalArgumentException(
+ "Cannot find RoutesBuilderLoader in classpath supporting file extension: " + extension);
}
+ return answer;
}
}
diff --git a/dsl/camel-java-joor-dsl/src/main/java/org/apache/camel/dsl/java/joor/JavaJoorClassLoader.java b/dsl/camel-java-joor-dsl/src/main/java/org/apache/camel/dsl/java/joor/JavaJoorClassLoader.java
index 0b3c08a26a8..95a5d6f3257 100644
--- a/dsl/camel-java-joor-dsl/src/main/java/org/apache/camel/dsl/java/joor/JavaJoorClassLoader.java
+++ b/dsl/camel-java-joor-dsl/src/main/java/org/apache/camel/dsl/java/joor/JavaJoorClassLoader.java
@@ -19,10 +19,7 @@ package org.apache.camel.dsl.java.joor;
import java.util.HashMap;
import java.util.Map;
-import org.apache.camel.CamelContext;
-import org.apache.camel.spi.CompilePostProcessor;
-
-public class JavaJoorClassLoader extends ClassLoader implements CompilePostProcessor {
+public class JavaJoorClassLoader extends ClassLoader {
private final Map<String, Class<?>> classes = new HashMap<>();
@@ -35,8 +32,7 @@ public class JavaJoorClassLoader extends ClassLoader implements CompilePostProce
return classes.get(name);
}
- @Override
- public void postCompile(CamelContext camelContext, String name, Class<?> clazz, byte[] byteCode, Object instance) {
+ public void addClass(String name, Class<?> clazz) {
if (name != null && clazz != null) {
classes.put(name, clazz);
}
diff --git a/dsl/camel-java-joor-dsl/src/main/java/org/apache/camel/dsl/java/joor/JavaRoutesBuilderLoader.java b/dsl/camel-java-joor-dsl/src/main/java/org/apache/camel/dsl/java/joor/JavaRoutesBuilderLoader.java
index a0c9e1e8abb..c96ff16c786 100644
--- a/dsl/camel-java-joor-dsl/src/main/java/org/apache/camel/dsl/java/joor/JavaRoutesBuilderLoader.java
+++ b/dsl/camel-java-joor-dsl/src/main/java/org/apache/camel/dsl/java/joor/JavaRoutesBuilderLoader.java
@@ -24,6 +24,8 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.camel.CamelContext;
import org.apache.camel.CamelContextAware;
@@ -52,6 +54,10 @@ public class JavaRoutesBuilderLoader extends ExtendedRouteBuilderLoaderSupport {
private static final Logger LOG = LoggerFactory.getLogger(JavaRoutesBuilderLoader.class);
+ private final ConcurrentMap<Collection<Resource>, CompilationUnit.Result> compiled = new ConcurrentHashMap<>();
+ private final Map<String, Resource> nameToResource = new HashMap<>();
+ private final JavaJoorClassLoader classLoader = new JavaJoorClassLoader();
+
public JavaRoutesBuilderLoader() {
super(EXTENSION);
}
@@ -63,15 +69,35 @@ public class JavaRoutesBuilderLoader extends ExtendedRouteBuilderLoaderSupport {
// register jOOR classloader to camel, so we are able to load classes we have compiled
CamelContext context = getCamelContext();
if (context != null) {
- JavaJoorClassLoader cl = new JavaJoorClassLoader();
- context.getClassResolver().addClassLoader(cl);
- addCompilePostProcessor(cl);
+ context.getClassResolver().addClassLoader(classLoader);
+ }
+ }
+
+ @Override
+ public void preParseRoute(Resource resource) throws Exception {
+ Collection<Resource> key = List.of(resource);
+ preParseRoutes(key);
+ }
+
+ @Override
+ public void preParseRoutes(Collection<Resource> resources) throws Exception {
+ CompilationUnit.Result result = compiled.get(resources);
+ if (result == null) {
+ result = compileResources(resources);
+ compiled.put(resources, result);
}
}
@Override
protected RouteBuilder doLoadRouteBuilder(Resource resource) throws Exception {
- Collection<RoutesBuilder> answer = doLoadRoutesBuilders(List.of(resource));
+ Collection<Resource> key = List.of(resource);
+ CompilationUnit.Result result = compiled.get(key);
+ if (result == null) {
+ result = compileResources(key);
+ compiled.put(key, result);
+ }
+
+ Collection<RoutesBuilder> answer = doLoadRoutesBuilders(key);
if (answer.size() == 1) {
RoutesBuilder builder = answer.iterator().next();
return (RouteBuilder) builder;
@@ -84,29 +110,10 @@ public class JavaRoutesBuilderLoader extends ExtendedRouteBuilderLoaderSupport {
protected Collection<RoutesBuilder> doLoadRoutesBuilders(Collection<Resource> resources) throws Exception {
Collection<RoutesBuilder> answer = new ArrayList<>();
- LOG.debug("Loading .java resources from: {}", resources);
-
- CompilationUnit unit = CompilationUnit.input();
-
- Map<String, Resource> nameToResource = new HashMap<>();
- for (Resource resource : resources) {
- try (InputStream is = resourceInputStream(resource)) {
- if (is == null) {
- throw new FileNotFoundException(resource.getLocation());
- }
- String content = IOHelper.loadText(is);
- String name = determineName(resource, content);
- unit.addClass(name, content);
- nameToResource.put(name, resource);
- }
- }
-
- LOG.debug("Compiling unit: {}", unit);
- CompilationUnit.Result result = MultiCompile.compileUnit(unit);
-
- // remember the last loaded resource-set if route reloading is enabled
- if (getCamelContext().hasService(RouteWatcherReloadStrategy.class) != null) {
- getCamelContext().getRegistry().bind(RouteWatcherReloadStrategy.RELOAD_RESOURCES, nameToResource.values());
+ // remove from pre-compiled
+ CompilationUnit.Result result = compiled.remove(resources);
+ if (result == null) {
+ result = compileResources(resources);
}
for (String className : result.getClassNames()) {
@@ -127,7 +134,7 @@ public class JavaRoutesBuilderLoader extends ExtendedRouteBuilderLoaderSupport {
// inject context and resource
CamelContextAware.trySetCamelContext(obj, getCamelContext());
- ResourceAware.trySetResource(obj, nameToResource.get(className));
+ ResourceAware.trySetResource(obj, nameToResource.remove(className));
}
} catch (Exception e) {
throw new RuntimeCamelException("Cannot create instance of class: " + className, e);
@@ -151,4 +158,42 @@ public class JavaRoutesBuilderLoader extends ExtendedRouteBuilderLoaderSupport {
return answer;
}
+ protected CompilationUnit.Result compileResources(Collection<Resource> resources) throws Exception {
+ LOG.debug("Loading .java resources from: {}", resources);
+
+ CompilationUnit unit = CompilationUnit.input();
+
+ for (Resource resource : resources) {
+ try (InputStream is = resourceInputStream(resource)) {
+ if (is == null) {
+ throw new FileNotFoundException(resource.getLocation());
+ }
+ String content = IOHelper.loadText(is);
+ String name = determineName(resource, content);
+ unit.addClass(name, content);
+ nameToResource.put(name, resource);
+ }
+ }
+
+ LOG.debug("Compiling unit: {}", unit);
+ CompilationUnit.Result result = MultiCompile.compileUnit(unit);
+
+ // remember the last loaded resource-set if route reloading is enabled
+ if (getCamelContext().hasService(RouteWatcherReloadStrategy.class) != null) {
+ getCamelContext().getRegistry().bind(RouteWatcherReloadStrategy.RELOAD_RESOURCES, nameToResource.values());
+ }
+
+ for (String className : result.getClassNames()) {
+ Class<?> clazz = result.getClass(className);
+ classLoader.addClass(className, clazz);
+ }
+
+ return result;
+ }
+
+ @Override
+ protected void doShutdown() throws Exception {
+ compiled.clear();
+ nameToResource.clear();
+ }
}
diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/test/groovy/org/apache/camel/dsl/yaml/IntegrationLoaderDependenciesTest.groovy b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/test/groovy/org/apache/camel/dsl/yaml/IntegrationLoaderDependenciesTest.groovy
index 81b42151e61..2f2ee3c62a0 100644
--- a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/test/groovy/org/apache/camel/dsl/yaml/IntegrationLoaderDependenciesTest.groovy
+++ b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/test/groovy/org/apache/camel/dsl/yaml/IntegrationLoaderDependenciesTest.groovy
@@ -21,7 +21,7 @@ import org.apache.camel.spi.DependencyStrategy
class IntegrationLoaderDependenciesTest extends YamlTestSupport {
- var List<String> deps = new ArrayList<>()
+ var Set<String> deps = new LinkedHashSet<>()
@Override
def doSetup() {