You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2021/03/13 16:18:30 UTC
[camel] branch master updated: CAMEL-16349: camel-core - MainHelper
load list from file in classpath to reduce memory
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/master by this push:
new 623ea19 CAMEL-16349: camel-core - MainHelper load list from file in classpath to reduce memory
623ea19 is described below
commit 623ea1974022f611b073d6c810af779e0b79aff3
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sat Mar 13 17:12:25 2021 +0100
CAMEL-16349: camel-core - MainHelper load list from file in classpath to reduce memory
---
core/camel-main/pom.xml | 23 +
.../org/apache/camel/main/components.properties | 347 ++++++++++++++
.../org/apache/camel/main/dataformats.properties | 46 ++
.../org/apache/camel/main/languages.properties | 20 +
.../org/apache/camel/main/BaseMainSupport.java | 47 +-
.../java/org/apache/camel/main/MainHelper.java | 503 +++------------------
.../java/org/apache/camel/main/MainHelperTest.java | 12 +-
.../camel/maven/packaging/UpdateMainHelper.java | 126 +-----
8 files changed, 538 insertions(+), 586 deletions(-)
diff --git a/core/camel-main/pom.xml b/core/camel-main/pom.xml
index 1121a41..7c30cf0 100644
--- a/core/camel-main/pom.xml
+++ b/core/camel-main/pom.xml
@@ -160,6 +160,29 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ <goal>add-resource</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/generated/java</source>
+ </sources>
+ <resources>
+ <resource>
+ <directory>src/generated/resources</directory>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/core/camel-main/src/generated/resources/org/apache/camel/main/components.properties b/core/camel-main/src/generated/resources/org/apache/camel/main/components.properties
new file mode 100644
index 0000000..a8bd19f
--- /dev/null
+++ b/core/camel-main/src/generated/resources/org/apache/camel/main/components.properties
@@ -0,0 +1,347 @@
+activemq
+ahc
+ahc-ws
+ahc-wss
+amqp
+apns
+arangodb
+as2
+asterisk
+atlasmap
+atmos
+atmosphere-websocket
+atom
+atomix-map
+atomix-messaging
+atomix-multimap
+atomix-queue
+atomix-set
+atomix-value
+avro
+aws-secrets-manager
+aws2-athena
+aws2-cw
+aws2-ddb
+aws2-ddbstream
+aws2-ec2
+aws2-ecs
+aws2-eks
+aws2-eventbridge
+aws2-iam
+aws2-kinesis
+aws2-kinesis-firehose
+aws2-kms
+aws2-lambda
+aws2-mq
+aws2-msk
+aws2-s3
+aws2-ses
+aws2-sns
+aws2-sqs
+aws2-sts
+aws2-translate
+azure-eventhubs
+azure-storage-blob
+azure-storage-datalake
+azure-storage-queue
+bean
+bean-validator
+beanstalk
+bonita
+box
+braintree
+browse
+caffeine-cache
+caffeine-loadcache
+chatscript
+chunk
+class
+cm-sms
+cmis
+coap
+coap+tcp
+coaps
+coaps+tcp
+cometd
+cometds
+consul
+controlbus
+corda
+couchbase
+couchdb
+cql
+cron
+crypto
+crypto-cms
+cxf
+cxfrs
+dataformat
+dataset
+dataset-test
+debezium-mongodb
+debezium-mysql
+debezium-postgres
+debezium-sqlserver
+digitalocean
+direct
+direct-vm
+disruptor
+disruptor-vm
+djl
+dns
+docker
+dozer
+drill
+dropbox
+ehcache
+elasticsearch-rest
+elsql
+etcd-keys
+etcd-stats
+etcd-watch
+exec
+facebook
+fhir
+file
+file-watch
+flatpack
+flink
+fop
+freemarker
+ftp
+ftps
+ganglia
+geocoder
+git
+github
+google-bigquery
+google-bigquery-sql
+google-calendar
+google-calendar-stream
+google-drive
+google-mail
+google-mail-stream
+google-pubsub
+google-sheets
+google-sheets-stream
+google-storage
+gora
+grape
+graphql
+grpc
+guava-eventbus
+hazelcast-atomicvalue
+hazelcast-instance
+hazelcast-list
+hazelcast-map
+hazelcast-multimap
+hazelcast-queue
+hazelcast-replicatedmap
+hazelcast-ringbuffer
+hazelcast-seda
+hazelcast-set
+hazelcast-topic
+hbase
+hdfs
+http
+https
+hwcloud-smn
+iec60870-client
+iec60870-server
+ignite-cache
+ignite-compute
+ignite-events
+ignite-idgen
+ignite-messaging
+ignite-queue
+ignite-set
+imap
+imaps
+infinispan
+infinispan-embedded
+influxdb
+iota
+ipfs
+irc
+ironmq
+jbpm
+jcache
+jclouds
+jcr
+jdbc
+jetty
+jgroups
+jgroups-raft
+jing
+jira
+jms
+jmx
+jolt
+jooq
+jpa
+jslt
+json-validator
+jsonata
+jt400
+kafka
+kamelet
+kubernetes-config-maps
+kubernetes-custom-resources
+kubernetes-deployments
+kubernetes-hpa
+kubernetes-job
+kubernetes-namespaces
+kubernetes-nodes
+kubernetes-persistent-volumes
+kubernetes-persistent-volumes-claims
+kubernetes-pods
+kubernetes-replication-controllers
+kubernetes-resources-quota
+kubernetes-secrets
+kubernetes-service-accounts
+kubernetes-services
+kudu
+language
+ldap
+ldif
+log
+lpr
+lucene
+lumberjack
+master
+metrics
+micrometer
+microprofile-metrics
+milo-client
+milo-server
+mina
+minio
+mllp
+mock
+mongodb
+mongodb-gridfs
+msv
+mustache
+mvel
+mybatis
+mybatis-bean
+nagios
+nats
+netty
+netty-http
+nitrite
+nsq
+oaipmh
+olingo2
+olingo4
+openshift-build-configs
+openshift-builds
+openstack-cinder
+openstack-glance
+openstack-keystone
+openstack-neutron
+openstack-nova
+openstack-swift
+optaplanner
+paho
+paho-mqtt5
+pdf
+pg-replication-slot
+pgevent
+platform-http
+pop3
+pop3s
+pubnub
+pulsar
+quartz
+quickfix
+rabbitmq
+reactive-streams
+ref
+rest
+rest-api
+rest-openapi
+rest-swagger
+resteasy
+robotframework
+rss
+saga
+salesforce
+sap-netweaver
+scheduler
+schematron
+scp
+seda
+service
+servicenow
+servlet
+sftp
+sip
+sips
+sjms
+sjms2
+slack
+smpp
+smpps
+smtp
+smtps
+snmp
+solr
+solrCloud
+solrs
+soroush
+spark
+splunk
+splunk-hec
+spring-batch
+spring-event
+spring-integration
+spring-ldap
+spring-rabbitmq
+spring-redis
+spring-ws
+sql
+sql-stored
+ssh
+stax
+stitch
+stomp
+stream
+string-template
+stub
+telegram
+thrift
+tika
+timer
+twilio
+twitter-directmessage
+twitter-search
+twitter-timeline
+undertow
+validator
+velocity
+vertx
+vertx-http
+vertx-kafka
+vertx-websocket
+vm
+weather
+web3j
+webhook
+websocket
+websocket-jsr356
+weka
+wordpress
+workday
+xchange
+xj
+xmlsecurity-sign
+xmlsecurity-verify
+xmpp
+xquery
+xslt
+xslt-saxon
+yammer
+zendesk
+zookeeper
+zookeeper-master
diff --git a/core/camel-main/src/generated/resources/org/apache/camel/main/dataformats.properties b/core/camel-main/src/generated/resources/org/apache/camel/main/dataformats.properties
new file mode 100644
index 0000000..7c52bd4
--- /dev/null
+++ b/core/camel-main/src/generated/resources/org/apache/camel/main/dataformats.properties
@@ -0,0 +1,46 @@
+any23
+asn1
+avro
+barcode
+base64
+beanio
+bindy-csv
+bindy-fixed
+bindy-kvp
+cbor
+crypto
+csv
+fhirJson
+fhirXml
+flatpack
+grok
+gzipdeflater
+hl7
+ical
+jacksonxml
+jaxb
+json-fastjson
+json-gson
+json-jackson
+json-johnzon
+json-jsonb
+json-xstream
+jsonApi
+lzf
+mime-multipart
+pgp
+protobuf
+rss
+secureXML
+soapjaxb
+syslog
+tarfile
+thrift
+tidyMarkup
+univocity-csv
+univocity-fixed
+univocity-tsv
+xstream
+yaml-snakeyaml
+zipdeflater
+zipfile
diff --git a/core/camel-main/src/generated/resources/org/apache/camel/main/languages.properties b/core/camel-main/src/generated/resources/org/apache/camel/main/languages.properties
new file mode 100644
index 0000000..81fba2e
--- /dev/null
+++ b/core/camel-main/src/generated/resources/org/apache/camel/main/languages.properties
@@ -0,0 +1,20 @@
+bean
+constant
+csimple
+datasonnet
+exchangeProperty
+file
+groovy
+header
+hl7terser
+joor
+jsonpath
+mvel
+ognl
+ref
+simple
+spel
+tokenize
+xpath
+xquery
+xtokenize
diff --git a/core/camel-main/src/main/java/org/apache/camel/main/BaseMainSupport.java b/core/camel-main/src/main/java/org/apache/camel/main/BaseMainSupport.java
index bb44dcb..04c6de9 100644
--- a/core/camel-main/src/main/java/org/apache/camel/main/BaseMainSupport.java
+++ b/core/camel-main/src/main/java/org/apache/camel/main/BaseMainSupport.java
@@ -68,7 +68,10 @@ import org.apache.camel.util.StringHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.camel.main.MainHelper.*;
+import static org.apache.camel.main.MainHelper.computeProperties;
+import static org.apache.camel.main.MainHelper.optionKey;
+import static org.apache.camel.main.MainHelper.setPropertiesOnTarget;
+import static org.apache.camel.main.MainHelper.validateOptionAndValue;
import static org.apache.camel.support.ObjectHelper.invokeMethod;
import static org.apache.camel.util.ReflectionHelper.findMethod;
import static org.apache.camel.util.StringHelper.matches;
@@ -83,6 +86,7 @@ public abstract class BaseMainSupport extends BaseService {
public static final String PROPERTY_PLACEHOLDER_LOCATION = "camel.main.property-placeholder-location";
private static final Logger LOG = LoggerFactory.getLogger(BaseMainSupport.class);
+ private final MainHelper helper;
protected final List<MainListener> listeners = new ArrayList<>();
protected volatile CamelContext camelContext;
protected MainConfigurationProperties mainConfigurationProperties = new MainConfigurationProperties();
@@ -94,10 +98,12 @@ public abstract class BaseMainSupport extends BaseService {
protected Properties overrideProperties;
protected BaseMainSupport() {
+ helper = new MainHelper();
}
protected BaseMainSupport(CamelContext camelContext) {
this.camelContext = camelContext;
+ helper = new MainHelper();
}
private static CamelSagaService resolveLraSagaService(CamelContext camelContext) throws Exception {
@@ -331,7 +337,8 @@ public abstract class BaseMainSupport extends BaseService {
String locations = propertyPlaceholderLocations;
if (locations == null) {
locations
- = lookupPropertyFromSysOrEnv(PROPERTY_PLACEHOLDER_LOCATION).orElse(defaultPropertyPlaceholderLocation);
+ = helper.lookupPropertyFromSysOrEnv(PROPERTY_PLACEHOLDER_LOCATION)
+ .orElse(defaultPropertyPlaceholderLocation);
}
if (locations != null) {
locations = locations.trim();
@@ -349,7 +356,7 @@ public abstract class BaseMainSupport extends BaseService {
Properties ip = initialProperties;
if (ip == null || ip.isEmpty()) {
- Optional<String> location = lookupPropertyFromSysOrEnv(INITIAL_PROPERTIES_LOCATION);
+ Optional<String> location = helper.lookupPropertyFromSysOrEnv(INITIAL_PROPERTIES_LOCATION);
if (location.isPresent()) {
try (InputStream is = ResourceHelper.resolveMandatoryResourceAsInputStream(camelContext, location.get())) {
ip = new Properties();
@@ -363,7 +370,7 @@ public abstract class BaseMainSupport extends BaseService {
Properties op = overrideProperties;
if (op == null || op.isEmpty()) {
- Optional<String> location = lookupPropertyFromSysOrEnv(OVERRIDE_PROPERTIES_LOCATION);
+ Optional<String> location = helper.lookupPropertyFromSysOrEnv(OVERRIDE_PROPERTIES_LOCATION);
if (location.isPresent()) {
try (InputStream is = ResourceHelper.resolveMandatoryResourceAsInputStream(camelContext, location.get())) {
op = new Properties();
@@ -419,7 +426,7 @@ public abstract class BaseMainSupport extends BaseService {
}
// we are now done with the main helper during bootstrap
- camelContext.adapt(ExtendedCamelContext.class).addBootstrap(MainHelper::bootstrapDone);
+ helper.bootstrapDone();
}
protected void configureStartupRecorder(CamelContext camelContext) {
@@ -579,7 +586,7 @@ public abstract class BaseMainSupport extends BaseService {
// load properties from ENV (override existing)
Properties propENV = null;
if (mainConfigurationProperties.isAutoConfigurationEnvironmentVariablesEnabled()) {
- propENV = loadEnvironmentVariablesAsProperties(new String[] { "camel.main." });
+ propENV = helper.loadEnvironmentVariablesAsProperties(new String[] { "camel.main." });
if (!propENV.isEmpty()) {
prop.putAll(propENV);
LOG.debug("Properties from OS environment variables:");
@@ -591,7 +598,7 @@ public abstract class BaseMainSupport extends BaseService {
// load properties from JVM (override existing)
Properties propJVM = null;
if (mainConfigurationProperties.isAutoConfigurationSystemPropertiesEnabled()) {
- propJVM = loadJvmSystemPropertiesAsProperties(new String[] { "camel.main." });
+ propJVM = helper.loadJvmSystemPropertiesAsProperties(new String[] { "camel.main." });
if (!propJVM.isEmpty()) {
prop.putAll(propJVM);
LOG.debug("Properties from JVM system properties:");
@@ -671,7 +678,7 @@ public abstract class BaseMainSupport extends BaseService {
// load properties from ENV (override existing)
if (mainConfigurationProperties.isAutoConfigurationEnvironmentVariablesEnabled()) {
- Properties propENV = loadEnvironmentVariablesAsProperties(new String[] { "camel.component.properties." });
+ Properties propENV = helper.loadEnvironmentVariablesAsProperties(new String[] { "camel.component.properties." });
if (!propENV.isEmpty()) {
prop.putAll(propENV);
LOG.debug("Properties from OS environment variables:");
@@ -682,7 +689,7 @@ public abstract class BaseMainSupport extends BaseService {
}
// load properties from JVM (override existing)
if (mainConfigurationProperties.isAutoConfigurationSystemPropertiesEnabled()) {
- Properties propJVM = loadJvmSystemPropertiesAsProperties(new String[] { "camel.component.properties." });
+ Properties propJVM = helper.loadJvmSystemPropertiesAsProperties(new String[] { "camel.component.properties." });
if (!propJVM.isEmpty()) {
prop.putAll(propJVM);
LOG.debug("Properties from JVM system properties:");
@@ -1051,14 +1058,14 @@ public abstract class BaseMainSupport extends BaseService {
// load properties from ENV (override existing)
if (mainConfigurationProperties.isAutoConfigurationEnvironmentVariablesEnabled()) {
- Properties propENV = loadEnvironmentVariablesAsProperties(new String[] { "camel.component.properties." });
+ Properties propENV = helper.loadEnvironmentVariablesAsProperties(new String[] { "camel.component.properties." });
if (!propENV.isEmpty()) {
prop.putAll(propENV);
}
}
// load properties from JVM (override existing)
if (mainConfigurationProperties.isAutoConfigurationSystemPropertiesEnabled()) {
- Properties propJVM = loadJvmSystemPropertiesAsProperties(new String[] { "camel.component.properties." });
+ Properties propJVM = helper.loadJvmSystemPropertiesAsProperties(new String[] { "camel.component.properties." });
if (!propJVM.isEmpty()) {
prop.putAll(propJVM);
}
@@ -1100,7 +1107,7 @@ public abstract class BaseMainSupport extends BaseService {
// load properties from ENV (override existing)
if (mainConfigurationProperties.isAutoConfigurationEnvironmentVariablesEnabled()) {
- Properties propENV = loadEnvironmentVariablesAsProperties(new String[] { "camel.main." });
+ Properties propENV = helper.loadEnvironmentVariablesAsProperties(new String[] { "camel.main." });
// special handling of these so remove them
// ENV variables cannot use dash so replace with dot
propENV.remove(INITIAL_PROPERTIES_LOCATION.replace('-', '.'));
@@ -1112,7 +1119,7 @@ public abstract class BaseMainSupport extends BaseService {
}
// load properties from JVM (override existing)
if (mainConfigurationProperties.isAutoConfigurationSystemPropertiesEnabled()) {
- Properties propJVM = loadJvmSystemPropertiesAsProperties(new String[] { "camel.main." });
+ Properties propJVM = helper.loadJvmSystemPropertiesAsProperties(new String[] { "camel.main." });
// special handling of these so remove them
propJVM.remove(INITIAL_PROPERTIES_LOCATION);
propJVM.remove(StringHelper.dashToCamelCase(INITIAL_PROPERTIES_LOCATION));
@@ -1190,15 +1197,15 @@ public abstract class BaseMainSupport extends BaseService {
// special configuration when using ENV variables as we need to extract the ENV variables
// that are for the out of the box components first, and then afterwards for any 3rd party custom components
Properties propENV = new OrderedProperties();
- MainHelper.addComponentEnvVariables(env, propENV, false);
- MainHelper.addDataFormatEnvVariables(env, propENV, false);
- MainHelper.addLanguageEnvVariables(env, propENV, false);
+ helper.addComponentEnvVariables(env, propENV, false);
+ helper.addDataFormatEnvVariables(env, propENV, false);
+ helper.addLanguageEnvVariables(env, propENV, false);
if (!env.isEmpty()) {
LOG.debug("Remaining {} ENV variables to configure custom components, dataformats, languages", env.size());
- MainHelper.addComponentEnvVariables(env, propENV, true);
- MainHelper.addDataFormatEnvVariables(env, propENV, true);
- MainHelper.addLanguageEnvVariables(env, propENV, true);
+ helper.addComponentEnvVariables(env, propENV, true);
+ helper.addDataFormatEnvVariables(env, propENV, true);
+ helper.addLanguageEnvVariables(env, propENV, true);
}
if (!propENV.isEmpty()) {
@@ -1207,7 +1214,7 @@ public abstract class BaseMainSupport extends BaseService {
}
// load properties from JVM (override existing)
if (mainConfigurationProperties.isAutoConfigurationSystemPropertiesEnabled()) {
- Properties propJVM = loadJvmSystemPropertiesAsProperties(
+ Properties propJVM = helper.loadJvmSystemPropertiesAsProperties(
new String[] { "camel.component.", "camel.dataformat.", "camel.language." });
if (!propJVM.isEmpty()) {
prop.putAll(propJVM);
diff --git a/core/camel-main/src/main/java/org/apache/camel/main/MainHelper.java b/core/camel-main/src/main/java/org/apache/camel/main/MainHelper.java
index 83fc400..0e115de 100644
--- a/core/camel-main/src/main/java/org/apache/camel/main/MainHelper.java
+++ b/core/camel-main/src/main/java/org/apache/camel/main/MainHelper.java
@@ -16,7 +16,11 @@
*/
package org.apache.camel.main;
-import java.util.Arrays;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -35,6 +39,7 @@ import org.apache.camel.spi.ExtendedPropertyConfigurerGetter;
import org.apache.camel.spi.PropertyConfigurer;
import org.apache.camel.support.PropertyBindingSupport;
import org.apache.camel.support.service.ServiceHelper;
+import org.apache.camel.util.IOHelper;
import org.apache.camel.util.ObjectHelper;
import org.apache.camel.util.OrderedProperties;
import org.apache.camel.util.StringHelper;
@@ -44,448 +49,33 @@ import org.slf4j.LoggerFactory;
public final class MainHelper {
private static final Logger LOG = LoggerFactory.getLogger(MainHelper.class);
- private static final Set<String> COMPONENT_ENV_NAMES = new HashSet<>(
- Arrays.asList(
- // Generated by camel build tools - do NOT edit this list!
- // COMPONENT-ENV-NAMES: START
- "ACTIVEMQ",
- "AHC",
- "AHC_WS",
- "AHC_WSS",
- "AMQP",
- "APNS",
- "ARANGODB",
- "AS2",
- "ASTERISK",
- "ATLASMAP",
- "ATMOS",
- "ATMOSPHERE_WEBSOCKET",
- "ATOM",
- "ATOMIX_MAP",
- "ATOMIX_MESSAGING",
- "ATOMIX_MULTIMAP",
- "ATOMIX_QUEUE",
- "ATOMIX_SET",
- "ATOMIX_VALUE",
- "AVRO",
- "AWS2_ATHENA",
- "AWS2_CW",
- "AWS2_DDB",
- "AWS2_DDBSTREAM",
- "AWS2_EC2",
- "AWS2_ECS",
- "AWS2_EKS",
- "AWS2_EVENTBRIDGE",
- "AWS2_IAM",
- "AWS2_KINESIS",
- "AWS2_KINESIS_FIREHOSE",
- "AWS2_KMS",
- "AWS2_LAMBDA",
- "AWS2_MQ",
- "AWS2_MSK",
- "AWS2_S3",
- "AWS2_SES",
- "AWS2_SNS",
- "AWS2_SQS",
- "AWS2_STS",
- "AWS2_TRANSLATE",
- "AWS_SECRETS_MANAGER",
- "AZURE_EVENTHUBS",
- "AZURE_STORAGE_BLOB",
- "AZURE_STORAGE_DATALAKE",
- "AZURE_STORAGE_QUEUE",
- "BEAN",
- "BEANSTALK",
- "BEAN_VALIDATOR",
- "BONITA",
- "BOX",
- "BRAINTREE",
- "BROWSE",
- "CAFFEINE_CACHE",
- "CAFFEINE_LOADCACHE",
- "CHATSCRIPT",
- "CHUNK",
- "CLASS",
- "CMIS",
- "CM_SMS",
- "COAP",
- "COAP+TCP",
- "COAPS",
- "COAPS+TCP",
- "COMETD",
- "COMETDS",
- "CONSUL",
- "CONTROLBUS",
- "CORDA",
- "COUCHBASE",
- "COUCHDB",
- "CQL",
- "CRON",
- "CRYPTO",
- "CRYPTO_CMS",
- "CXF",
- "CXFRS",
- "DATAFORMAT",
- "DATASET",
- "DATASET_TEST",
- "DEBEZIUM_MONGODB",
- "DEBEZIUM_MYSQL",
- "DEBEZIUM_POSTGRES",
- "DEBEZIUM_SQLSERVER",
- "DIGITALOCEAN",
- "DIRECT",
- "DIRECT_VM",
- "DISRUPTOR",
- "DISRUPTOR_VM",
- "DJL",
- "DNS",
- "DOCKER",
- "DOZER",
- "DRILL",
- "DROPBOX",
- "EHCACHE",
- "ELASTICSEARCH_REST",
- "ELSQL",
- "ETCD_KEYS",
- "ETCD_STATS",
- "ETCD_WATCH",
- "EXEC",
- "FACEBOOK",
- "FHIR",
- "FILE",
- "FILE_WATCH",
- "FLATPACK",
- "FLINK",
- "FOP",
- "FREEMARKER",
- "FTP",
- "FTPS",
- "GANGLIA",
- "GEOCODER",
- "GIT",
- "GITHUB",
- "GOOGLE_BIGQUERY",
- "GOOGLE_BIGQUERY_SQL",
- "GOOGLE_CALENDAR",
- "GOOGLE_CALENDAR_STREAM",
- "GOOGLE_DRIVE",
- "GOOGLE_MAIL",
- "GOOGLE_MAIL_STREAM",
- "GOOGLE_PUBSUB",
- "GOOGLE_SHEETS",
- "GOOGLE_SHEETS_STREAM",
- "GOOGLE_STORAGE",
- "GORA",
- "GRAPE",
- "GRAPHQL",
- "GRPC",
- "GUAVA_EVENTBUS",
- "HAZELCAST_ATOMICVALUE",
- "HAZELCAST_INSTANCE",
- "HAZELCAST_LIST",
- "HAZELCAST_MAP",
- "HAZELCAST_MULTIMAP",
- "HAZELCAST_QUEUE",
- "HAZELCAST_REPLICATEDMAP",
- "HAZELCAST_RINGBUFFER",
- "HAZELCAST_SEDA",
- "HAZELCAST_SET",
- "HAZELCAST_TOPIC",
- "HBASE",
- "HDFS",
- "HTTP",
- "HTTPS",
- "HWCLOUD_SMN",
- "IEC60870_CLIENT",
- "IEC60870_SERVER",
- "IGNITE_CACHE",
- "IGNITE_COMPUTE",
- "IGNITE_EVENTS",
- "IGNITE_IDGEN",
- "IGNITE_MESSAGING",
- "IGNITE_QUEUE",
- "IGNITE_SET",
- "IMAP",
- "IMAPS",
- "INFINISPAN",
- "INFINISPAN_EMBEDDED",
- "INFLUXDB",
- "IOTA",
- "IPFS",
- "IRC",
- "IRONMQ",
- "JBPM",
- "JCACHE",
- "JCLOUDS",
- "JCR",
- "JDBC",
- "JETTY",
- "JGROUPS",
- "JGROUPS_RAFT",
- "JING",
- "JIRA",
- "JMS",
- "JMX",
- "JOLT",
- "JOOQ",
- "JPA",
- "JSLT",
- "JSONATA",
- "JSON_VALIDATOR",
- "JT400",
- "KAFKA",
- "KAMELET",
- "KUBERNETES_CONFIG_MAPS",
- "KUBERNETES_CUSTOM_RESOURCES",
- "KUBERNETES_DEPLOYMENTS",
- "KUBERNETES_HPA",
- "KUBERNETES_JOB",
- "KUBERNETES_NAMESPACES",
- "KUBERNETES_NODES",
- "KUBERNETES_PERSISTENT_VOLUMES",
- "KUBERNETES_PERSISTENT_VOLUMES_CLAIMS",
- "KUBERNETES_PODS",
- "KUBERNETES_REPLICATION_CONTROLLERS",
- "KUBERNETES_RESOURCES_QUOTA",
- "KUBERNETES_SECRETS",
- "KUBERNETES_SERVICES",
- "KUBERNETES_SERVICE_ACCOUNTS",
- "KUDU",
- "LANGUAGE",
- "LDAP",
- "LDIF",
- "LOG",
- "LPR",
- "LUCENE",
- "LUMBERJACK",
- "MASTER",
- "METRICS",
- "MICROMETER",
- "MICROPROFILE_METRICS",
- "MILO_CLIENT",
- "MILO_SERVER",
- "MINA",
- "MINIO",
- "MLLP",
- "MOCK",
- "MONGODB",
- "MONGODB_GRIDFS",
- "MSV",
- "MUSTACHE",
- "MVEL",
- "MYBATIS",
- "MYBATIS_BEAN",
- "NAGIOS",
- "NATS",
- "NETTY",
- "NETTY_HTTP",
- "NITRITE",
- "NSQ",
- "OAIPMH",
- "OLINGO2",
- "OLINGO4",
- "OPENSHIFT_BUILDS",
- "OPENSHIFT_BUILD_CONFIGS",
- "OPENSTACK_CINDER",
- "OPENSTACK_GLANCE",
- "OPENSTACK_KEYSTONE",
- "OPENSTACK_NEUTRON",
- "OPENSTACK_NOVA",
- "OPENSTACK_SWIFT",
- "OPTAPLANNER",
- "PAHO",
- "PAHO_MQTT5",
- "PDF",
- "PGEVENT",
- "PG_REPLICATION_SLOT",
- "PLATFORM_HTTP",
- "POP3",
- "POP3S",
- "PUBNUB",
- "PULSAR",
- "QUARTZ",
- "QUICKFIX",
- "RABBITMQ",
- "REACTIVE_STREAMS",
- "REF",
- "REST",
- "RESTEASY",
- "REST_API",
- "REST_OPENAPI",
- "REST_SWAGGER",
- "ROBOTFRAMEWORK",
- "RSS",
- "SAGA",
- "SALESFORCE",
- "SAP_NETWEAVER",
- "SCHEDULER",
- "SCHEMATRON",
- "SCP",
- "SEDA",
- "SERVICE",
- "SERVICENOW",
- "SERVLET",
- "SFTP",
- "SIP",
- "SIPS",
- "SJMS",
- "SJMS2",
- "SLACK",
- "SMPP",
- "SMPPS",
- "SMTP",
- "SMTPS",
- "SNMP",
- "SOLR",
- "SOLRCLOUD",
- "SOLRS",
- "SOROUSH",
- "SPARK",
- "SPLUNK",
- "SPLUNK_HEC",
- "SPRING_BATCH",
- "SPRING_EVENT",
- "SPRING_INTEGRATION",
- "SPRING_LDAP",
- "SPRING_RABBITMQ",
- "SPRING_REDIS",
- "SPRING_WS",
- "SQL",
- "SQL_STORED",
- "SSH",
- "STAX",
- "STITCH",
- "STOMP",
- "STREAM",
- "STRING_TEMPLATE",
- "STUB",
- "TELEGRAM",
- "THRIFT",
- "TIKA",
- "TIMER",
- "TWILIO",
- "TWITTER_DIRECTMESSAGE",
- "TWITTER_SEARCH",
- "TWITTER_TIMELINE",
- "UNDERTOW",
- "VALIDATOR",
- "VELOCITY",
- "VERTX",
- "VERTX_HTTP",
- "VERTX_KAFKA",
- "VERTX_WEBSOCKET",
- "VM",
- "WEATHER",
- "WEB3J",
- "WEBHOOK",
- "WEBSOCKET",
- "WEBSOCKET_JSR356",
- "WEKA",
- "WORDPRESS",
- "WORKDAY",
- "XCHANGE",
- "XJ",
- "XMLSECURITY_SIGN",
- "XMLSECURITY_VERIFY",
- "XMPP",
- "XQUERY",
- "XSLT",
- "XSLT_SAXON",
- "YAMMER",
- "ZENDESK",
- "ZOOKEEPER",
- "ZOOKEEPER_MASTER"
- // COMPONENT-ENV-NAMES: END
- ));
-
- private static final Set<String> DATAFORMAT_ENV_NAMES = new HashSet<>(
- Arrays.asList(
- // Generated by camel build tools - do NOT edit this list!
- // DATAFORMAT-ENV-NAMES: START
- "ANY23",
- "ASN1",
- "AVRO",
- "BARCODE",
- "BASE64",
- "BEANIO",
- "BINDY_CSV",
- "BINDY_FIXED",
- "BINDY_KVP",
- "CBOR",
- "CRYPTO",
- "CSV",
- "FHIRJSON",
- "FHIRXML",
- "FLATPACK",
- "GROK",
- "GZIPDEFLATER",
- "HL7",
- "ICAL",
- "JACKSONXML",
- "JAXB",
- "JSONAPI",
- "JSON_FASTJSON",
- "JSON_GSON",
- "JSON_JACKSON",
- "JSON_JOHNZON",
- "JSON_JSONB",
- "JSON_XSTREAM",
- "LZF",
- "MIME_MULTIPART",
- "PGP",
- "PROTOBUF",
- "RSS",
- "SECUREXML",
- "SOAPJAXB",
- "SYSLOG",
- "TARFILE",
- "THRIFT",
- "TIDYMARKUP",
- "UNIVOCITY_CSV",
- "UNIVOCITY_FIXED",
- "UNIVOCITY_TSV",
- "XSTREAM",
- "YAML_SNAKEYAML",
- "ZIPDEFLATER",
- "ZIPFILE"
- // DATAFORMAT-ENV-NAMES: END
- ));
-
- private static final Set<String> LANGUAGE_ENV_NAMES = new HashSet<>(
- Arrays.asList(
- // Generated by camel build tools - do NOT edit this list!
- // LANGUAGE-ENV-NAMES: START
- "BEAN",
- "CONSTANT",
- "CSIMPLE",
- "DATASONNET",
- "EXCHANGEPROPERTY",
- "FILE",
- "GROOVY",
- "HEADER",
- "HL7TERSER",
- "JOOR",
- "JSONPATH",
- "MVEL",
- "OGNL",
- "REF",
- "SIMPLE",
- "SPEL",
- "TOKENIZE",
- "XPATH",
- "XQUERY",
- "XTOKENIZE"
- // LANGUAGE-ENV-NAMES: END
- ));
-
- private MainHelper() {
+ private final Set<String> componentEnvNames = new HashSet<>();
+ private final Set<String> dataformatEnvNames = new HashSet<>();
+ private final Set<String> languageEnvNames = new HashSet<>();
+
+ public MainHelper() {
+ try {
+ InputStream is = MainHelper.class.getResourceAsStream("/org/apache/camel/main/components.properties");
+ loadLines(is, componentEnvNames, s -> "CAMEL_COMPONENT_" + s.toUpperCase(Locale.US).replace('-', '_'));
+ IOHelper.close(is);
+
+ is = MainHelper.class.getResourceAsStream("/org/apache/camel/main/dataformats.properties");
+ loadLines(is, dataformatEnvNames, s -> "CAMEL_DATAFORMAT_" + s.toUpperCase(Locale.US).replace('-', '_'));
+ IOHelper.close(is);
+
+ is = MainHelper.class.getResourceAsStream("/org/apache/camel/main/languages.properties");
+ loadLines(is, languageEnvNames, s -> "CAMEL_LANGUAGE_" + s.toUpperCase(Locale.US).replace('-', '_'));
+ IOHelper.close(is);
+ } catch (Exception e) {
+ throw new RuntimeException("Error loading catalog information from classpath", e);
+ }
}
- public static void bootstrapDone() {
+ public void bootstrapDone() {
// after bootstrap then these maps are no longer needed
- COMPONENT_ENV_NAMES.clear();
- DATAFORMAT_ENV_NAMES.clear();
- LANGUAGE_ENV_NAMES.clear();
+ componentEnvNames.clear();
+ dataformatEnvNames.clear();
+ languageEnvNames.clear();
}
public static String toEnvVar(String name) {
@@ -535,7 +125,7 @@ public final class MainHelper {
return answer;
}
- public static void addComponentEnvVariables(Map<String, String> env, Properties properties, boolean custom) {
+ public void addComponentEnvVariables(Map<String, String> env, Properties properties, boolean custom) {
Set<String> toRemove = new HashSet<>();
env.forEach((k, v) -> {
if (custom) {
@@ -545,7 +135,7 @@ public final class MainHelper {
properties.put(ck, v);
} else {
Optional<String> e
- = COMPONENT_ENV_NAMES.stream().map(s -> "CAMEL_COMPONENT_" + s).filter(k::startsWith).findFirst();
+ = componentEnvNames.stream().filter(k::startsWith).findFirst();
if (e.isPresent()) {
toRemove.add(k);
String cname = "camel.component." + e.get().substring(16).toLowerCase(Locale.US).replace('_', '-');
@@ -557,7 +147,7 @@ public final class MainHelper {
toRemove.forEach(env::remove);
}
- public static void addDataFormatEnvVariables(Map<String, String> env, Properties properties, boolean custom) {
+ public void addDataFormatEnvVariables(Map<String, String> env, Properties properties, boolean custom) {
Set<String> toRemove = new HashSet<>();
env.forEach((k, v) -> {
if (custom) {
@@ -567,7 +157,7 @@ public final class MainHelper {
properties.put(ck, v);
} else {
Optional<String> e
- = DATAFORMAT_ENV_NAMES.stream().map(s -> "CAMEL_DATAFORMAT_" + s).filter(k::startsWith).findFirst();
+ = dataformatEnvNames.stream().filter(k::startsWith).findFirst();
if (e.isPresent()) {
toRemove.add(k);
String cname = "camel.dataformat." + e.get().substring(17).toLowerCase(Locale.US).replace('_', '-');
@@ -579,7 +169,7 @@ public final class MainHelper {
toRemove.forEach(env::remove);
}
- public static void addLanguageEnvVariables(Map<String, String> env, Properties properties, boolean custom) {
+ public void addLanguageEnvVariables(Map<String, String> env, Properties properties, boolean custom) {
Set<String> toRemove = new HashSet<>();
env.forEach((k, v) -> {
if (custom) {
@@ -589,7 +179,7 @@ public final class MainHelper {
properties.put(ck, v);
} else {
Optional<String> e
- = LANGUAGE_ENV_NAMES.stream().map(s -> "CAMEL_LANGUAGE_" + s).filter(k::startsWith).findFirst();
+ = languageEnvNames.stream().filter(k::startsWith).findFirst();
if (e.isPresent()) {
toRemove.add(k);
String cname = "camel.language." + e.get().substring(15).toLowerCase(Locale.US).replace('_', '-');
@@ -601,10 +191,6 @@ public final class MainHelper {
toRemove.forEach(env::remove);
}
- private static String asEnvName(String name) {
- return name.toUpperCase(Locale.US).replace('-', '_');
- }
-
public static Properties loadJvmSystemPropertiesAsProperties(String[] prefixes) {
Properties answer = new OrderedProperties();
if (prefixes == null || prefixes.length == 0) {
@@ -826,4 +412,21 @@ public final class MainHelper {
}
}
+ /**
+ * Loads the entire stream into memory as a String and returns it.
+ * <p/>
+ * <b>Notice:</b> This implementation appends a <tt>\n</tt> as line terminator at the of the text.
+ * <p/>
+ * Warning, don't use for crazy big streams :)
+ */
+ private static void loadLines(InputStream in, Set<String> lines, Function<String, String> func) throws IOException {
+ try (final InputStreamReader isr = new InputStreamReader(in);
+ final BufferedReader reader = new LineNumberReader(isr)) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ lines.add(func.apply(line));
+ }
+ }
+ }
+
}
diff --git a/core/camel-main/src/test/java/org/apache/camel/main/MainHelperTest.java b/core/camel-main/src/test/java/org/apache/camel/main/MainHelperTest.java
index 4469aff..0770c5e 100644
--- a/core/camel-main/src/test/java/org/apache/camel/main/MainHelperTest.java
+++ b/core/camel-main/src/test/java/org/apache/camel/main/MainHelperTest.java
@@ -25,12 +25,14 @@ import org.junit.jupiter.api.Test;
public class MainHelperTest {
+ private MainHelper helper = new MainHelper();
+
@Test
public void testAddComponentEnvVariables() {
Map<String, String> env = MainHelper.filterEnvVariables(new String[] { "CAMEL_COMPONENT_" });
env.put("CAMEL_COMPONENT_AWS2_S3_ACCESS_KEY", "mysecretkey");
Properties prop = new OrderedProperties();
- MainHelper.addComponentEnvVariables(env, prop, false);
+ helper.addComponentEnvVariables(env, prop, false);
Assertions.assertEquals(0, env.size());
Assertions.assertEquals(1, prop.size());
@@ -43,7 +45,7 @@ public class MainHelperTest {
env.put("CAMEL_DATAFORMAT_BASE64_LINE_LENGTH", "64");
env.put("CAMEL_DATAFORMAT_JACKSONXML_PRETTYPRINT", "true");
Properties prop = new OrderedProperties();
- MainHelper.addDataFormatEnvVariables(env, prop, false);
+ helper.addDataFormatEnvVariables(env, prop, false);
Assertions.assertEquals(0, env.size());
Assertions.assertEquals(2, prop.size());
@@ -56,7 +58,7 @@ public class MainHelperTest {
Map<String, String> env = MainHelper.filterEnvVariables(new String[] { "CAMEL_LANGUAGE_" });
env.put("CAMEL_LANGUAGE_JOOR_PRE_COMPILE", "false");
Properties prop = new OrderedProperties();
- MainHelper.addLanguageEnvVariables(env, prop, false);
+ helper.addLanguageEnvVariables(env, prop, false);
Assertions.assertEquals(0, env.size());
Assertions.assertEquals(1, prop.size());
@@ -70,13 +72,13 @@ public class MainHelperTest {
env.put("CAMEL_COMPONENT_FOO_VERBOSE", "true");
env.put("CAMEL_COMPONENT_FOO_PRETTY_PRINT", "false");
Properties prop = new OrderedProperties();
- MainHelper.addComponentEnvVariables(env, prop, false);
+ helper.addComponentEnvVariables(env, prop, false);
Assertions.assertEquals(2, env.size());
Assertions.assertEquals(1, prop.size());
Assertions.assertEquals("mysecretkey", prop.getProperty("camel.component.aws2-s3.access-key"));
- MainHelper.addComponentEnvVariables(env, prop, true);
+ helper.addComponentEnvVariables(env, prop, true);
Assertions.assertEquals(0, env.size());
Assertions.assertEquals(3, prop.size());
diff --git a/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/UpdateMainHelper.java b/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/UpdateMainHelper.java
index c464f2f..706c858 100644
--- a/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/UpdateMainHelper.java
+++ b/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/UpdateMainHelper.java
@@ -17,23 +17,8 @@
package org.apache.camel.maven.packaging;
import java.io.File;
-import java.nio.file.Path;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.StringJoiner;
-import java.util.TreeSet;
-import java.util.stream.Collectors;
-import org.apache.camel.tooling.model.ComponentModel;
-import org.apache.camel.tooling.model.DataFormatModel;
-import org.apache.camel.tooling.model.JsonMapper;
-import org.apache.camel.tooling.model.LanguageModel;
-import org.apache.camel.tooling.util.PackageHelper;
-import org.apache.camel.tooling.util.Strings;
-import org.apache.camel.util.json.JsonObject;
-import org.apache.camel.util.json.Jsoner;
+import org.apache.camel.tooling.util.FileUtil;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugins.annotations.Mojo;
@@ -42,20 +27,13 @@ import org.apache.maven.plugins.annotations.Parameter;
import static org.apache.camel.tooling.util.PackageHelper.findCamelDirectory;
/**
- * Updates the MainHelper.java with the known component, data-format, and language names.
+ * Updates camel-main with the known component, data-format, and language names.
*/
@Mojo(name = "update-main-helper", threadSafe = true)
public class UpdateMainHelper extends AbstractGeneratorMojo {
- private static final String COMPONENT_START_TOKEN = "// COMPONENT-ENV-NAMES: START";
- private static final String COMPONENT_END_TOKEN = "// COMPONENT-ENV-NAMES: END";
- private static final String DATAFORMAT_START_TOKEN = "// DATAFORMAT-ENV-NAMES: START";
- private static final String DATAFORMAT_END_TOKEN = "// DATAFORMAT-ENV-NAMES: END";
- private static final String LANGUAGE_START_TOKEN = "// LANGUAGE-ENV-NAMES: START";
- private static final String LANGUAGE_END_TOKEN = "// LANGUAGE-ENV-NAMES: END";
-
@Parameter(defaultValue = "${project.basedir}/src/generated/resources/org/apache/camel/catalog/")
- protected File jsonDir;
+ protected File propertiesDir;
@Parameter(defaultValue = "${project.basedir}/")
protected File baseDir;
@@ -74,98 +52,24 @@ public class UpdateMainHelper extends AbstractGeneratorMojo {
return;
}
- List<Path> jsonFiles = PackageHelper.findJsonFiles(jsonDir.toPath()).collect(Collectors.toList());
- Set<String> components = new TreeSet<>();
- Set<String> dataformats = new TreeSet<>();
- Set<String> languages = new TreeSet<>();
-
- for (Path file : jsonFiles) {
- final String name = PackageHelper.asName(file);
-
- try {
- String json = PackageHelper.loadText(file.toFile());
- JsonObject obj = (JsonObject) Jsoner.deserialize(json);
-
- Map<String, Object> model;
- boolean isComponent = (model = obj.getMap("component")) != null;
- boolean isDataFormat = !isComponent && (model = obj.getMap("dataformat")) != null;
- boolean isLanguage = !isComponent && !isDataFormat && (model = obj.getMap("language")) != null;
-
- // only check these kind
- if (!isComponent && !isDataFormat && !isLanguage) {
- continue;
- }
-
- if (isComponent) {
- ComponentModel cm = JsonMapper.generateComponentModel(json);
- components.add(asEnvName(cm.getScheme()));
- if (cm.getAlternativeSchemes() != null) {
- String[] aliases = cm.getAlternativeSchemes().split(",");
- for (String alias : aliases) {
- components.add(asEnvName(alias));
- }
- }
- } else if (isDataFormat) {
- DataFormatModel dm = JsonMapper.generateDataFormatModel(json);
- dataformats.add(asEnvName(dm.getName()));
- } else if (isLanguage) {
- LanguageModel lm = JsonMapper.generateLanguageModel(json);
- languages.add(asEnvName(lm.getName()));
- }
- } catch (Exception e) {
- throw new MojoExecutionException("Error loading json: " + name, e);
- }
- }
-
try {
- boolean updated = updateMainHelper(camelDir, components, COMPONENT_START_TOKEN, COMPONENT_END_TOKEN);
- updated |= updateMainHelper(camelDir, dataformats, DATAFORMAT_START_TOKEN, DATAFORMAT_END_TOKEN);
- updated |= updateMainHelper(camelDir, languages, LANGUAGE_START_TOKEN, LANGUAGE_END_TOKEN);
- if (updated) {
- getLog().info("Updated camel-main/src/main/java/org/apache/camel/main/MainHelper.java file");
- } else {
- getLog().debug("No changes to camel-main/src/main/java/org/apache/camel/main/MainHelper.java file");
- }
+ File targetDir = new File(camelDir, "src/generated/resources/org/apache/camel/main");
- } catch (Exception e) {
- throw new MojoExecutionException("Error updating MainHelper.java", e);
- }
- }
+ File source = new File(propertiesDir, "components.properties");
+ File target = new File(targetDir, "components.properties");
+ FileUtil.updateFile(source.toPath(), target.toPath());
- private boolean updateMainHelper(File camelDir, Set<String> names, String startToken, String endToken) throws Exception {
- // load source code and update
- File java = new File(camelDir, "src/main/java/org/apache/camel/main/MainHelper.java");
- String text = PackageHelper.loadText(java);
- String spaces20 = " ";
- String spaces12 = " ";
+ source = new File(propertiesDir, "dataformats.properties");
+ target = new File(targetDir, "dataformats.properties");
+ FileUtil.updateFile(source.toPath(), target.toPath());
- StringJoiner sb = new StringJoiner(",\n");
- for (String name : names) {
- sb.add(spaces20 + "\"" + name + "\"");
- }
- String changed = sb.toString();
+ source = new File(propertiesDir, "languages.properties");
+ target = new File(targetDir, "languages.properties");
+ FileUtil.updateFile(source.toPath(), target.toPath());
- String existing = Strings.between(text, startToken, endToken);
- if (existing != null) {
- // remove leading line breaks etc
- existing = existing.trim();
- changed = changed.trim();
- if (existing.equals(changed)) {
- return false;
- } else {
- String before = Strings.before(text, startToken);
- String after = Strings.after(text, endToken);
- text = before + startToken + "\n" + spaces20 + changed + "\n" + spaces12 + endToken + after;
- PackageHelper.writeText(java, text);
- return true;
- }
+ } catch (Exception e) {
+ throw new MojoExecutionException("Error updating camel-main", e);
}
-
- return false;
- }
-
- private static String asEnvName(String name) {
- return name.toUpperCase(Locale.US).replace('-', '_');
}
}