You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2022/04/20 09:46:59 UTC
[camel] 01/02: CAMEL-17991: camel-jbang - camel init - download source from github to local
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 2f959d19c01c0f8970b5d22878875b2fc43279bd
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Wed Apr 20 11:37:58 2022 +0200
CAMEL-17991: camel-jbang - camel init - download source from github to local
---
.../dsl/jbang/core/commands/GitHubHelper.java | 153 +++++++++++++++++++++
.../apache/camel/dsl/jbang/core/commands/Init.java | 53 +++++++
.../apache/camel/dsl/jbang/core/commands/Run.java | 117 +---------------
3 files changed, 213 insertions(+), 110 deletions(-)
diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/GitHubHelper.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/GitHubHelper.java
new file mode 100644
index 00000000000..efd708d5656
--- /dev/null
+++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/GitHubHelper.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dsl.jbang.core.commands;
+
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.time.Duration;
+import java.util.StringJoiner;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.camel.util.AntPathMatcher;
+import org.apache.camel.util.FileUtil;
+
+public final class GitHubHelper {
+
+ private GitHubHelper() {
+ }
+
+ public static String asGithubSingleUrl(String url) throws Exception {
+ // strip https://github.com/
+ url = url.substring(19);
+ // https://github.com/apache/camel-k/blob/main/examples/languages/routes.kts
+ // https://raw.githubusercontent.com/apache/camel-kamelets-examples/main/jbang/hello-java/Hey.java
+ // https://github.com/apache/camel-kamelets-examples/blob/main/jbang/hello-java/Hey.java
+ url = url.replaceFirst("/", ":");
+ url = url.replaceFirst("/", ":");
+ url = url.replaceFirst("tree/", "");
+ url = url.replaceFirst("blob/", "");
+ url = url.replaceFirst("/", ":");
+ return "github:" + url;
+ }
+
+ public static void fetchGithubUrls(String url, StringJoiner all) throws Exception {
+ doFetchGithubUrls(url, null, null, null, all);
+ }
+
+ public static void fetchGithubUrls(String url, StringJoiner routes, StringJoiner kamelets, StringJoiner properties)
+ throws Exception {
+ doFetchGithubUrls(url, routes, kamelets, properties, null);
+ }
+
+ private static void doFetchGithubUrls(
+ String url, StringJoiner routes, StringJoiner kamelets, StringJoiner properties,
+ StringJoiner all)
+ throws Exception {
+
+ // this is a directory, so we need to query github which files are there and filter them
+
+ // URL: https://api.github.com/repos/apache/camel-k/contents/examples/kamelets/kameletbindings
+ // URL: https://api.github.com/repos/apache/camel-k/contents/examples/kamelets/kameletbindings?ref=v1.7.0
+ // https://github.com/apache/camel-k/tree/main/examples/kamelets/kameletbindings
+ // https://github.com/apache/camel-k/tree/v1.7.0/examples/kamelets/kameletbindings
+
+ // strip https://github.com/
+ url = url.substring(19);
+
+ String[] parts = url.split("/");
+ if (parts.length < 5) {
+ return;
+ }
+
+ String org = parts[0];
+ String repo = parts[1];
+ String action = parts[2];
+ String branch = parts[3];
+ String path;
+ String wildcard = null;
+ StringJoiner sj = new StringJoiner("/");
+ for (int i = 4; i < parts.length; i++) {
+ if (i == parts.length - 1) {
+ // last element uses wildcard to filter which files to include
+ if (parts[i].contains("*")) {
+ wildcard = parts[i];
+ break;
+ }
+ }
+ sj.add(parts[i]);
+ }
+ path = sj.toString();
+
+ if ("tree".equals(action)) {
+ // https://api.github.com/repos/apache/camel-k/contents/examples/kamelets/kameletbindings?ref=v1.7.0
+ url = "https://api.github.com/repos/" + org + "/" + repo + "/contents/" + path;
+ if (!"main".equals(branch) && !"master".equals(branch)) {
+ url = url + "?ref=" + branch;
+ }
+ }
+
+ resolveGithubAsRawFiles(url, wildcard, routes, kamelets, properties, all);
+ }
+
+ private static void resolveGithubAsRawFiles(
+ String url, String wildcard, StringJoiner routes, StringJoiner kamelets, StringJoiner properties, StringJoiner all)
+ throws Exception {
+
+ // use JDK http client to call github api
+ HttpClient hc = HttpClient.newHttpClient();
+ HttpResponse<String> res = hc.send(HttpRequest.newBuilder(new URI(url)).timeout(Duration.ofSeconds(20)).build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ if (res.statusCode() == 200) {
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode root = mapper.readTree(res.body());
+ for (JsonNode c : root) {
+ String name = c.get("name").asText();
+ String ext = FileUtil.onlyExt(name, false);
+ boolean match = wildcard == null || AntPathMatcher.INSTANCE.match(wildcard, name, false);
+ if (match) {
+ if (kamelets != null && "kamelet.yaml".equalsIgnoreCase(ext)) {
+ String htmlUrl = c.get("html_url").asText();
+ String u = asGithubSingleUrl(htmlUrl);
+ kamelets.add(u);
+ } else if (properties != null && "properties".equalsIgnoreCase(ext)) {
+ String htmlUrl = c.get("html_url").asText();
+ String u = asGithubSingleUrl(htmlUrl);
+ properties.add(u);
+ } else if (routes != null) {
+ if ("java".equalsIgnoreCase(ext) || "xml".equalsIgnoreCase(ext)
+ || "yaml".equalsIgnoreCase(ext)
+ || "groovy".equalsIgnoreCase(ext) || "js".equalsIgnoreCase(ext) || "jsh".equalsIgnoreCase(ext)
+ || "kts".equalsIgnoreCase(ext)) {
+ String htmlUrl = c.get("html_url").asText();
+ String u = asGithubSingleUrl(htmlUrl);
+ routes.add(u);
+ }
+ } else if (all != null) {
+ String htmlUrl = c.get("html_url").asText();
+ String u = asGithubSingleUrl(htmlUrl);
+ all.add(u);
+ }
+ }
+ }
+ }
+ }
+
+}
diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Init.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Init.java
index 1d5913333fb..1445ed1052d 100644
--- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Init.java
+++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Init.java
@@ -18,14 +18,24 @@ package org.apache.camel.dsl.jbang.core.commands;
import java.io.FileOutputStream;
import java.io.InputStream;
+import java.util.StringJoiner;
import java.util.concurrent.Callable;
+import org.apache.camel.CamelContext;
+import org.apache.camel.dsl.jbang.core.common.exceptions.ResourceDoesNotExist;
+import org.apache.camel.github.GitHubResourceResolver;
+import org.apache.camel.impl.lw.LightweightCamelContext;
+import org.apache.camel.spi.Resource;
import org.apache.camel.util.FileUtil;
import org.apache.camel.util.IOHelper;
+import org.apache.commons.io.IOUtils;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
+import static org.apache.camel.dsl.jbang.core.commands.GitHubHelper.asGithubSingleUrl;
+import static org.apache.camel.dsl.jbang.core.commands.GitHubHelper.fetchGithubUrls;
+
@Command(name = "init", description = "Initialize empty Camel integrations")
class Init implements Callable<Integer> {
@@ -43,6 +53,13 @@ class Init implements Callable<Integer> {
@Override
public Integer call() throws Exception {
+
+ // is the file referring to an existing file on github
+ // then we should download the file to local for use
+ if (file.startsWith("https://github.com/")) {
+ return downloadFromGithub();
+ }
+
String ext = FileUtil.onlyExt(file, false);
if ("yaml".equals(ext) && integration) {
ext = "integration.yaml";
@@ -61,4 +78,40 @@ class Init implements Callable<Integer> {
IOHelper.writeText(context, new FileOutputStream(file, false));
return 0;
}
+
+ private int downloadFromGithub() throws Exception {
+ StringJoiner all = new StringJoiner(",");
+
+ String ext = FileUtil.onlyExt(file);
+ boolean wildcard = FileUtil.onlyName(file, false).contains("*");
+ if (ext != null && !wildcard) {
+ // it is a single file so map to
+ String url = asGithubSingleUrl(file);
+ all.add(url);
+ } else {
+ fetchGithubUrls(file, all);
+ }
+
+ if (all.length() > 0) {
+ CamelContext tiny = new LightweightCamelContext();
+ GitHubResourceResolver resolver = new GitHubResourceResolver();
+ resolver.setCamelContext(tiny);
+ for (String u : all.toString().split(",")) {
+ Resource resource = resolver.resolve(u);
+ if (!resource.exists()) {
+ throw new ResourceDoesNotExist(resource);
+ }
+
+ String loc = resource.getLocation();
+ String name = FileUtil.stripPath(loc);
+
+ try (FileOutputStream fo = new FileOutputStream(name)) {
+ IOUtils.copy(resource.getInputStream(), fo);
+ }
+ }
+ }
+
+ return 0;
+ }
+
}
diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Run.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Run.java
index 4b138a8c9cd..cb4c6f75cc5 100644
--- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Run.java
+++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Run.java
@@ -18,12 +18,7 @@ package org.apache.camel.dsl.jbang.core.commands;
import java.io.File;
import java.io.IOException;
-import java.net.URI;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
import java.nio.file.FileSystems;
-import java.time.Duration;
import java.util.Arrays;
import java.util.Locale;
import java.util.StringJoiner;
@@ -32,13 +27,10 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.camel.CamelContext;
import org.apache.camel.dsl.jbang.core.common.RuntimeUtil;
import org.apache.camel.main.KameletMain;
import org.apache.camel.support.ResourceHelper;
-import org.apache.camel.util.AntPathMatcher;
import org.apache.camel.util.FileUtil;
import org.apache.camel.util.ObjectHelper;
import org.apache.camel.util.StringHelper;
@@ -46,6 +38,9 @@ import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import picocli.CommandLine.Parameters;
+import static org.apache.camel.dsl.jbang.core.commands.GitHubHelper.asGithubSingleUrl;
+import static org.apache.camel.dsl.jbang.core.commands.GitHubHelper.fetchGithubUrls;
+
@Command(name = "run", description = "Run Camel")
class Run implements Callable<Integer> {
@@ -304,13 +299,13 @@ class Run implements Callable<Integer> {
// it is a single file so map to
file = asGithubSingleUrl(file);
} else {
- StringJoiner files = new StringJoiner(",");
+ StringJoiner routes = new StringJoiner(",");
StringJoiner kamelets = new StringJoiner(",");
StringJoiner properties = new StringJoiner(",");
- fetchGithubUrls(file, files, kamelets, properties);
+ fetchGithubUrls(file, routes, kamelets, properties);
- if (files.length() > 0) {
- file = files.toString();
+ if (routes.length() > 0) {
+ file = routes.toString();
}
if (properties.length() > 0) {
main.addInitialProperty("camel.component.properties.location", properties.toString());
@@ -405,104 +400,6 @@ class Run implements Callable<Integer> {
return lockFile;
}
- private static String asGithubSingleUrl(String url) throws Exception {
- // strip https://github.com/
- url = url.substring(19);
- // https://github.com/apache/camel-k/blob/main/examples/languages/routes.kts
- // https://raw.githubusercontent.com/apache/camel-kamelets-examples/main/jbang/hello-java/Hey.java
- // https://github.com/apache/camel-kamelets-examples/blob/main/jbang/hello-java/Hey.java
- url = url.replaceFirst("/", ":");
- url = url.replaceFirst("/", ":");
- url = url.replaceFirst("tree/", "");
- url = url.replaceFirst("blob/", "");
- url = url.replaceFirst("/", ":");
- return "github:" + url;
- }
-
- private static void fetchGithubUrls(String url, StringJoiner files, StringJoiner kamelets, StringJoiner properties)
- throws Exception {
- // this is a directory, so we need to query github which files are there and filter them
-
- // URL: https://api.github.com/repos/apache/camel-k/contents/examples/kamelets/kameletbindings
- // URL: https://api.github.com/repos/apache/camel-k/contents/examples/kamelets/kameletbindings?ref=v1.7.0
- // https://github.com/apache/camel-k/tree/main/examples/kamelets/kameletbindings
- // https://github.com/apache/camel-k/tree/v1.7.0/examples/kamelets/kameletbindings
-
- // strip https://github.com/
- url = url.substring(19);
-
- String[] parts = url.split("/");
- if (parts.length < 5) {
- return;
- }
-
- String org = parts[0];
- String repo = parts[1];
- String action = parts[2];
- String branch = parts[3];
- String path;
- String wildcard = null;
- StringJoiner sj = new StringJoiner("/");
- for (int i = 4; i < parts.length; i++) {
- if (i == parts.length - 1) {
- // last element uses wildcard to filter which files to include
- if (parts[i].contains("*")) {
- wildcard = parts[i];
- break;
- }
- }
- sj.add(parts[i]);
- }
- path = sj.toString();
-
- if ("tree".equals(action)) {
- // https://api.github.com/repos/apache/camel-k/contents/examples/kamelets/kameletbindings?ref=v1.7.0
- url = "https://api.github.com/repos/" + org + "/" + repo + "/contents/" + path;
- if (!"main".equals(branch) && !"master".equals(branch)) {
- url = url + "?ref=" + branch;
- }
- }
-
- downloadGithubFiles(url, wildcard, files, kamelets, properties);
- }
-
- private static void downloadGithubFiles(
- String url, String wildcard, StringJoiner files, StringJoiner kamelets, StringJoiner properties)
- throws Exception {
-
- // use JDK http client to call github api
- HttpClient hc = HttpClient.newHttpClient();
- HttpResponse<String> res = hc.send(HttpRequest.newBuilder(new URI(url)).timeout(Duration.ofSeconds(20)).build(),
- HttpResponse.BodyHandlers.ofString());
-
- if (res.statusCode() == 200) {
- ObjectMapper mapper = new ObjectMapper();
- JsonNode root = mapper.readTree(res.body());
- for (JsonNode c : root) {
- String name = c.get("name").asText();
- String ext = FileUtil.onlyExt(name, false);
- boolean match = wildcard == null || AntPathMatcher.INSTANCE.match(wildcard, name, false);
- if (match) {
- if ("kamelet.yaml".equalsIgnoreCase(ext)) {
- String htmlUrl = c.get("html_url").asText();
- String u = asGithubSingleUrl(htmlUrl);
- kamelets.add(u);
- } else if ("properties".equalsIgnoreCase(ext)) {
- String htmlUrl = c.get("html_url").asText();
- String u = asGithubSingleUrl(htmlUrl);
- properties.add(u);
- } else if ("java".equalsIgnoreCase(ext) || "xml".equalsIgnoreCase(ext) || "yaml".equalsIgnoreCase(ext)
- || "groovy".equalsIgnoreCase(ext) || "js".equalsIgnoreCase(ext) || "jsh".equalsIgnoreCase(ext)
- || "kts".equalsIgnoreCase(ext)) {
- String htmlUrl = c.get("html_url").asText();
- String u = asGithubSingleUrl(htmlUrl);
- files.add(u);
- }
- }
- }
- }
- }
-
private boolean knownFile(String file) {
String ext = FileUtil.onlyExt(file, true);
if (ext != null) {