You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2022/03/19 14:15:21 UTC
[camel] branch main updated (784684f -> 6832fc2)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git.
from 784684f Regen for commit 3fe499dcb71a63bf5245a4282702af328c9548b1 (#7235)
new 2f97665 CAMEL-17815: camel-jbang - In modeline mode then eager discover routes and pre-load modeline to allow properties to be used during bootstrap.
new 72dcb90 CAMEL-17815: camel-jbang - In modeline mode then eager discover routes and pre-load modeline to allow properties to be used during bootstrap.
new 6832fc2 CAMEL-17815: camel-jbang - In modeline mode then eager discover routes and pre-load spec/* from camel-k integration yaml files to allow properties to be used during bootstrap.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/camel/spi/RoutesBuilderLoader.java | 12 +++
.../java/org/apache/camel/spi/RoutesLoader.java | 4 +
.../camel/impl/engine/DefaultRoutesLoader.java | 38 ++++++---
.../org/apache/camel/main/BaseMainSupport.java | 3 +-
.../org/apache/camel/main/RoutesConfigurer.java | 12 ++-
.../IntegrationConfigurationPropertiesSource.java | 12 ++-
.../camel/dsl/yaml/YamlRoutesBuilderLoader.java | 99 +++++++++++++++++-----
.../dsl/yaml/YamlRoutesBuilderLoaderSupport.java | 6 +-
8 files changed, 145 insertions(+), 41 deletions(-)
[camel] 01/03: CAMEL-17815: camel-jbang - In modeline mode then eager discover routes and pre-load modeline to allow properties to be used during bootstrap.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 2f976651ec8f43e2cc92c544e47d29d9287e189d
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sat Mar 19 08:32:05 2022 +0100
CAMEL-17815: camel-jbang - In modeline mode then eager discover routes and pre-load modeline to allow properties to be used during bootstrap.
---
.../src/main/java/org/apache/camel/main/BaseMainSupport.java | 3 +--
.../src/main/java/org/apache/camel/main/RoutesConfigurer.java | 4 ++--
2 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/core/camel-main/src/main/java/org/apache/camel/main/BaseMainSupport.java b/core/camel-main/src/main/java/org/apache/camel/main/BaseMainSupport.java
index 27c0aa5..cf7cb55 100644
--- a/core/camel-main/src/main/java/org/apache/camel/main/BaseMainSupport.java
+++ b/core/camel-main/src/main/java/org/apache/camel/main/BaseMainSupport.java
@@ -680,8 +680,7 @@ public abstract class BaseMainSupport extends BaseService {
}
protected void autoConfigurationRoutesIncludePattern(
- CamelContext camelContext, Map<String, String> autoConfiguredProperties)
- throws Exception {
+ CamelContext camelContext, Map<String, String> autoConfiguredProperties) {
Object pattern = getInitialProperties().getProperty("camel.main.routesIncludePattern");
if (pattern != null) {
diff --git a/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java b/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java
index 9e979ef..e680485 100644
--- a/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java
+++ b/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java
@@ -251,8 +251,8 @@ public class RoutesConfigurer {
try {
LOG.debug("RoutesCollectorEnabled: {}", getRoutesCollector());
- // add discovered routes from directories
- StopWatch watch = new StopWatch();
+ // we can only scan for modeline for routes that we can load from directory as modelines
+ // are comments in the source files
Collection<RoutesBuilder> routesFromDirectory = getRoutesCollector().collectRoutesFromDirectory(
camelContext,
getRoutesExcludePattern(),
[camel] 03/03: CAMEL-17815: camel-jbang - In modeline mode then eager discover routes and pre-load spec/* from camel-k integration yaml files to allow properties to be used during bootstrap.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 6832fc27c6648d7757255ee0fd75cd889997d20e
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sat Mar 19 13:59:13 2022 +0100
CAMEL-17815: camel-jbang - In modeline mode then eager discover routes and pre-load spec/* from camel-k integration yaml files to allow properties to be used during bootstrap.
---
.../org/apache/camel/spi/RoutesBuilderLoader.java | 12 +++
.../java/org/apache/camel/spi/RoutesLoader.java | 4 +
.../camel/impl/engine/DefaultRoutesLoader.java | 38 ++++++---
.../org/apache/camel/main/RoutesConfigurer.java | 8 ++
.../camel/dsl/yaml/YamlRoutesBuilderLoader.java | 99 +++++++++++++++++-----
.../dsl/yaml/YamlRoutesBuilderLoaderSupport.java | 6 +-
6 files changed, 132 insertions(+), 35 deletions(-)
diff --git a/core/camel-api/src/main/java/org/apache/camel/spi/RoutesBuilderLoader.java b/core/camel-api/src/main/java/org/apache/camel/spi/RoutesBuilderLoader.java
index f55da10..7492e03 100644
--- a/core/camel-api/src/main/java/org/apache/camel/spi/RoutesBuilderLoader.java
+++ b/core/camel-api/src/main/java/org/apache/camel/spi/RoutesBuilderLoader.java
@@ -49,4 +49,16 @@ public interface RoutesBuilderLoader extends StaticService, CamelContextAware {
* @return a {@link RoutesBuilder}
*/
RoutesBuilder loadRoutesBuilder(Resource resource) throws Exception;
+
+ /**
+ * Pre-parses the {@link RoutesBuilder} from {@link Resource}.
+ *
+ * This is used during bootstrap, to eager detect configurations from route DSL resources which makes it possible to
+ * specify configurations that affect the bootstrap, such as by camel-jbang and camel-yaml-dsl.
+ *
+ * @param resource the resource to be pre parsed.
+ */
+ default void preParseRoute(Resource resource) throws Exception {
+ // noop
+ }
}
diff --git a/core/camel-api/src/main/java/org/apache/camel/spi/RoutesLoader.java b/core/camel-api/src/main/java/org/apache/camel/spi/RoutesLoader.java
index 4ac5c35..e00720e 100644
--- a/core/camel-api/src/main/java/org/apache/camel/spi/RoutesLoader.java
+++ b/core/camel-api/src/main/java/org/apache/camel/spi/RoutesLoader.java
@@ -115,4 +115,8 @@ public interface RoutesLoader extends CamelContextAware {
* @return a collection {@link RoutesBuilder}
*/
Collection<RoutesBuilder> findRoutesBuilders(Collection<Resource> resources) throws Exception;
+
+ default void preParseRoute(Resource resource) throws Exception {
+ // noop
+ }
}
diff --git a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRoutesLoader.java b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRoutesLoader.java
index 1d134e3..0f4737f 100644
--- a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRoutesLoader.java
+++ b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRoutesLoader.java
@@ -87,24 +87,14 @@ public class DefaultRoutesLoader extends ServiceSupport implements RoutesLoader,
List<RoutesBuilder> answer = new ArrayList<>(resources.size());
for (Resource resource : resources) {
- // the loader to use is derived from the file extension
- final String extension = FileUtil.onlyExt(resource.getLocation(), false);
-
- if (ObjectHelper.isEmpty(extension)) {
- throw new IllegalArgumentException(
- "Unable to determine file extension for resource: " + resource.getLocation());
- }
-
- RoutesBuilderLoader loader = getRoutesLoader(extension);
- if (loader == null) {
- throw new IllegalArgumentException(
- "Cannot find RoutesBuilderLoader in classpath supporting file extension: " + extension);
- }
+ RoutesBuilderLoader loader = resolveRoutesBuilderLoader(resource);
if (camelContext.isModeline()) {
ModelineFactory factory = camelContext.adapt(ExtendedCamelContext.class).getModelineFactory();
// gather resources for modeline
factory.parseModeline(resource);
+ // pre-parse before loading
+ loader.preParseRoute(resource);
}
RoutesBuilder builder = loader.loadRoutesBuilder(resource);
@@ -116,6 +106,11 @@ public class DefaultRoutesLoader extends ServiceSupport implements RoutesLoader,
return answer;
}
+ @Override
+ public void preParseRoute(Resource resource) throws Exception {
+ resolveRoutesBuilderLoader(resource).preParseRoute(resource);
+ }
+
/**
* Looks up a {@link RoutesBuilderLoader} in the registry or fallback to a factory finder mechanism if none found.
*
@@ -178,4 +173,21 @@ public class DefaultRoutesLoader extends ServiceSupport implements RoutesLoader,
return answer;
}
+ protected RoutesBuilderLoader resolveRoutesBuilderLoader(Resource resource) throws Exception {
+ // the loader to use is derived from the file extension
+ final String extension = FileUtil.onlyExt(resource.getLocation(), false);
+
+ if (ObjectHelper.isEmpty(extension)) {
+ throw new IllegalArgumentException(
+ "Unable to determine file extension for resource: " + resource.getLocation());
+ }
+
+ RoutesBuilderLoader loader = getRoutesLoader(extension);
+ if (loader == null) {
+ throw new IllegalArgumentException(
+ "Cannot find RoutesBuilderLoader in classpath supporting file extension: " + extension);
+ }
+ return loader;
+ }
+
}
diff --git a/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java b/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java
index e680485..e05a0af 100644
--- a/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java
+++ b/core/camel-main/src/main/java/org/apache/camel/main/RoutesConfigurer.java
@@ -30,6 +30,7 @@ import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.spi.CamelBeanPostProcessor;
import org.apache.camel.spi.ModelineFactory;
import org.apache.camel.spi.Resource;
+import org.apache.camel.spi.RoutesLoader;
import org.apache.camel.support.OrderedComparator;
import org.apache.camel.util.StopWatch;
import org.apache.camel.util.TimeUtils;
@@ -282,6 +283,13 @@ public class RoutesConfigurer {
LOG.debug("Parsing modeline: {}", resource);
factory.parseModeline(resource);
}
+ // the resource may also have additional configurations which we need to detect via pre-parsing
+ for (Resource resource : resources) {
+ LOG.debug("Pre-parsing: {}", resource);
+ RoutesLoader loader = camelContext.adapt(ExtendedCamelContext.class).getRoutesLoader();
+ loader.preParseRoute(resource);
+ }
+
}
}
diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/main/java/org/apache/camel/dsl/yaml/YamlRoutesBuilderLoader.java b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/main/java/org/apache/camel/dsl/yaml/YamlRoutesBuilderLoader.java
index f43e373..3d6d141 100644
--- a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/main/java/org/apache/camel/dsl/yaml/YamlRoutesBuilderLoader.java
+++ b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/main/java/org/apache/camel/dsl/yaml/YamlRoutesBuilderLoader.java
@@ -16,6 +16,8 @@
*/
package org.apache.camel.dsl.yaml;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
@@ -59,11 +61,18 @@ import org.apache.camel.support.ObjectHelper;
import org.apache.camel.support.PropertyBindingSupport;
import org.apache.camel.util.FileUtil;
import org.apache.camel.util.URISupport;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.snakeyaml.engine.v2.api.YamlUnicodeReader;
+import org.snakeyaml.engine.v2.composer.Composer;
import org.snakeyaml.engine.v2.nodes.MappingNode;
import org.snakeyaml.engine.v2.nodes.Node;
import org.snakeyaml.engine.v2.nodes.NodeTuple;
import org.snakeyaml.engine.v2.nodes.NodeType;
import org.snakeyaml.engine.v2.nodes.SequenceNode;
+import org.snakeyaml.engine.v2.parser.Parser;
+import org.snakeyaml.engine.v2.parser.ParserImpl;
+import org.snakeyaml.engine.v2.scanner.StreamReader;
import static org.apache.camel.dsl.yaml.common.YamlDeserializerSupport.asMap;
import static org.apache.camel.dsl.yaml.common.YamlDeserializerSupport.asMappingNode;
@@ -79,6 +88,8 @@ public class YamlRoutesBuilderLoader extends YamlRoutesBuilderLoaderSupport {
public static final String EXTENSION = "yaml";
+ private static final Logger LOG = LoggerFactory.getLogger(YamlRoutesBuilderLoader.class);
+
// API versions for Camel-K Integration and Kamelet Binding
// we are lenient so lets just assume we can work with any of the v1 even if they evolve
private static final String INTEGRATION_VERSION = "camel.apache.org/v1";
@@ -104,7 +115,7 @@ public class YamlRoutesBuilderLoader extends YamlRoutesBuilderLoaderSupport {
ctx.setResource(resource);
setDeserializationContext(root, ctx);
- Object target = preConfigureNode(root, ctx);
+ Object target = preConfigureNode(root, ctx, false);
if (target == null) {
return;
}
@@ -196,7 +207,7 @@ public class YamlRoutesBuilderLoader extends YamlRoutesBuilderLoaderSupport {
ctx.setResource(resource);
setDeserializationContext(root, ctx);
- Object target = preConfigureNode(root, ctx);
+ Object target = preConfigureNode(root, ctx, false);
if (target == null) {
return;
}
@@ -236,7 +247,7 @@ public class YamlRoutesBuilderLoader extends YamlRoutesBuilderLoaderSupport {
};
}
- private Object preConfigureNode(Node root, YamlDeserializationContext ctx) throws Exception {
+ private Object preConfigureNode(Node root, YamlDeserializationContext ctx, boolean preParse) throws Exception {
Object target = root;
// check if the yaml is a camel-k yaml with embedded binding/routes (called flow(s))
@@ -249,8 +260,9 @@ public class YamlRoutesBuilderLoader extends YamlRoutesBuilderLoaderSupport {
boolean binding = anyTupleMatches(mn.getValue(), "apiVersion", v -> v.startsWith(BINDING_VERSION)) &&
anyTupleMatches(mn.getValue(), "kind", "KameletBinding");
if (integration) {
- target = preConfigureIntegration(root, ctx, target);
- } else if (binding) {
+ target = preConfigureIntegration(root, ctx, target, preParse);
+ } else if (binding && !preParse) {
+ // kamelet binding does not take part in pre-parse phase
target = preConfigureKameletBinding(root, ctx, target);
}
}
@@ -261,7 +273,9 @@ public class YamlRoutesBuilderLoader extends YamlRoutesBuilderLoaderSupport {
/**
* Camel K Integration file
*/
- private Object preConfigureIntegration(Node root, YamlDeserializationContext ctx, Object target) {
+ private Object preConfigureIntegration(Node root, YamlDeserializationContext ctx, Object target, boolean preParse) {
+ // when in pre-parse phase then we only want to gather spec/dependencies,spec/configuration,spec/traits
+
List<Object> answer = new ArrayList<>();
// if there are dependencies then include them first
@@ -270,6 +284,7 @@ public class YamlRoutesBuilderLoader extends YamlRoutesBuilderLoaderSupport {
var dep = preConfigureDependencies(deps);
answer.add(dep);
}
+
// if there are configurations then include them early
Node configuration = nodeAt(root, "/spec/configuration");
if (configuration != null) {
@@ -288,20 +303,24 @@ public class YamlRoutesBuilderLoader extends YamlRoutesBuilderLoaderSupport {
var list = preConfigureTraitEnvironment(configuration);
answer.addAll(list);
}
- // if there are sources then include them before routes
- Node sources = nodeAt(root, "/spec/sources");
- if (sources != null) {
- var list = preConfigureSources(sources);
- answer.addAll(list);
- }
- // add routes last
- Node routes = nodeAt(root, "/spec/flows");
- if (routes == null) {
- routes = nodeAt(root, "/spec/flow");
- }
- if (routes != null) {
- answer.add(routes);
+
+ if (!preParse) {
+ // if there are sources then include them before routes
+ Node sources = nodeAt(root, "/spec/sources");
+ if (sources != null) {
+ var list = preConfigureSources(sources);
+ answer.addAll(list);
+ }
+ // add routes last
+ Node routes = nodeAt(root, "/spec/flows");
+ if (routes == null) {
+ routes = nodeAt(root, "/spec/flow");
+ }
+ if (routes != null) {
+ answer.add(routes);
+ }
}
+
return answer;
}
@@ -624,4 +643,46 @@ public class YamlRoutesBuilderLoader extends YamlRoutesBuilderLoaderSupport {
}
}
+ @Override
+ public void preParseRoute(Resource resource) throws Exception {
+ LOG.trace("Pre-parsing: {}", resource.getLocation());
+
+ if (!resource.exists()) {
+ throw new FileNotFoundException("Resource not found: " + resource.getLocation());
+ }
+
+ try (InputStream is = resource.getInputStream()) {
+ final StreamReader reader = new StreamReader(settings, new YamlUnicodeReader(is));
+ final Parser parser = new ParserImpl(settings, reader);
+ final Composer composer = new Composer(settings, parser);
+
+ composer.getSingleNode()
+ .map(node -> preParseNode(node, resource));
+ }
+ }
+
+ private Object preParseNode(Node root, Resource resource) {
+ LOG.trace("Pre-parsing node: {}", root);
+
+ YamlDeserializationContext ctx = getDeserializationContext();
+ ctx.setResource(resource);
+ setDeserializationContext(root, ctx);
+
+ try {
+ Object target = preConfigureNode(root, ctx, true);
+ Iterator<?> it = ObjectHelper.createIterator(target);
+ while (it.hasNext()) {
+ target = it.next();
+ if (target instanceof CamelContextCustomizer) {
+ CamelContextCustomizer customizer = (CamelContextCustomizer) target;
+ customizer.configure(getCamelContext());
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeCamelException("Error pre-parsing resource: " + resource.getLocation(), e);
+ }
+
+ return null;
+ }
+
}
diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/main/java/org/apache/camel/dsl/yaml/YamlRoutesBuilderLoaderSupport.java b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/main/java/org/apache/camel/dsl/yaml/YamlRoutesBuilderLoaderSupport.java
index 6102197..04429d2 100644
--- a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/main/java/org/apache/camel/dsl/yaml/YamlRoutesBuilderLoaderSupport.java
+++ b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/main/java/org/apache/camel/dsl/yaml/YamlRoutesBuilderLoaderSupport.java
@@ -50,9 +50,9 @@ import static org.apache.camel.dsl.yaml.common.YamlDeserializerSupport.asText;
public abstract class YamlRoutesBuilderLoaderSupport extends RouteBuilderLoaderSupport {
public static final String DESERIALIZATION_MODE = "CamelYamlDslDeserializationMode";
- private LoadSettings settings;
- private YamlDeserializationContext deserializationContext;
- private YamlDeserializationMode deserializationMode;
+ LoadSettings settings;
+ YamlDeserializationContext deserializationContext;
+ YamlDeserializationMode deserializationMode;
public YamlRoutesBuilderLoaderSupport(String extension) {
super(extension);
[camel] 02/03: CAMEL-17815: camel-jbang - In modeline mode then eager discover routes and pre-load modeline to allow properties to be used during bootstrap.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 72dcb90f8a862f6cfeb8cd4a67f86c788bcc9ab6
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sat Mar 19 08:36:41 2022 +0100
CAMEL-17815: camel-jbang - In modeline mode then eager discover routes and pre-load modeline to allow properties to be used during bootstrap.
---
.../dsl/yaml/IntegrationConfigurationPropertiesSource.java | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/main/java/org/apache/camel/dsl/yaml/IntegrationConfigurationPropertiesSource.java b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/main/java/org/apache/camel/dsl/yaml/IntegrationConfigurationPropertiesSource.java
index b420507..14d270f 100644
--- a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/main/java/org/apache/camel/dsl/yaml/IntegrationConfigurationPropertiesSource.java
+++ b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/main/java/org/apache/camel/dsl/yaml/IntegrationConfigurationPropertiesSource.java
@@ -63,7 +63,7 @@ public class IntegrationConfigurationPropertiesSource implements PropertiesSourc
if (line.contains("=")) {
String key = StringHelper.before(line, "=").trim();
String value = StringHelper.after(line, "=").trim();
- properties.setProperty(key, value);
+ setProperty(key, value);
} else {
if (ResourceHelper.hasScheme(line)) {
// it is a properties file so load resource
@@ -74,7 +74,7 @@ public class IntegrationConfigurationPropertiesSource implements PropertiesSourc
String v = prop.getProperty(k);
String key = k.trim();
String value = v.trim();
- properties.setProperty(key, value);
+ setProperty(key, value);
}
} catch (Exception e) {
// ignore
@@ -83,6 +83,14 @@ public class IntegrationConfigurationPropertiesSource implements PropertiesSourc
}
}
+ protected void setProperty(String key, String value) {
+ properties.setProperty(key, value);
+ if (!camelContext.isStarted()) {
+ // if we are bootstrapping then also set as initial property, so it can be used there as well
+ camelContext.getPropertiesComponent().addInitialProperty(key, value);
+ }
+ }
+
@Override
public String toString() {
return "camel-yaml-dsl";