You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ma...@apache.org on 2023/03/01 21:44:53 UTC
[camel-karavan] 02/02: Fix #645
This is an automated email from the ASF dual-hosted git repository.
marat pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-karavan.git
commit ec37f5fa1337f3fc7407a4e1afc3cda04c97b4ff
Author: Marat Gubaidullin <ma...@gmail.com>
AuthorDate: Wed Mar 1 16:44:40 2023 -0500
Fix #645
---
karavan-app/pom.xml | 17 +-
.../{GitResource.java => ProjectGitResource.java} | 21 +-
.../org/apache/camel/karavan/model/CommitInfo.java | 39 ++++
.../apache/camel/karavan/service/GitService.java | 214 ++++++++++++++++-----
.../camel/karavan/service/InfinispanService.java | 22 ++-
.../camel/karavan/service/KaravanService.java | 5 +-
.../camel/karavan/service/KubernetesService.java | 4 +-
.../{ImportService.java => ProjectService.java} | 139 +++++++------
.../apache/camel/karavan/service/ServiceUtil.java | 66 +++++++
.../src/main/resources/application.properties | 6 +-
karavan-app/src/main/webui/src/api/KaravanApi.tsx | 12 +-
.../main/webui/src/projects/ProjectPageToolbar.tsx | 7 +-
12 files changed, 403 insertions(+), 149 deletions(-)
diff --git a/karavan-app/pom.xml b/karavan-app/pom.xml
index 402b8cde..151cfb34 100644
--- a/karavan-app/pom.xml
+++ b/karavan-app/pom.xml
@@ -30,8 +30,9 @@
<quarkus.platform.artifact-id>quarkus-bom</quarkus.platform.artifact-id>
<quarkus.platform.group-id>io.quarkus.platform</quarkus.platform.group-id>
<quarkus.platform.version>2.16.0.Final</quarkus.platform.version>
- <surefire-plugin.version>3.0.0-M5</surefire-plugin.version>
+ <camel-quarkus.version>2.16.0</camel-quarkus.version>
<camel.version>3.20.2</camel.version>
+ <surefire-plugin.version>3.0.0-M5</surefire-plugin.version>
<infinispan.version>14.0.5.Final</infinispan.version>
<tekton.version>6.2.0</tekton.version>
<jgit.version>2.3.1</jgit.version>
@@ -114,6 +115,10 @@
<groupId>io.quarkus</groupId>
<artifactId>quarkus-qute</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.quarkus</groupId>
+ <artifactId>quarkus-scheduler</artifactId>
+ </dependency>
<dependency>
<groupId>io.fabric8</groupId>
<artifactId>tekton-client</artifactId>
@@ -152,11 +157,11 @@
<artifactId>rest-assured</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>io.quarkiverse.quinoa</groupId>
- <artifactId>quarkus-quinoa</artifactId>
- <version>${quinoa.version}</version>
- </dependency>
+<!-- <dependency>-->
+<!-- <groupId>io.quarkiverse.quinoa</groupId>-->
+<!-- <artifactId>quarkus-quinoa</artifactId>-->
+<!-- <version>${quinoa.version}</version>-->
+<!-- </dependency>-->
</dependencies>
<build>
<resources>
diff --git a/karavan-app/src/main/java/org/apache/camel/karavan/api/GitResource.java b/karavan-app/src/main/java/org/apache/camel/karavan/api/ProjectGitResource.java
similarity index 72%
rename from karavan-app/src/main/java/org/apache/camel/karavan/api/GitResource.java
rename to karavan-app/src/main/java/org/apache/camel/karavan/api/ProjectGitResource.java
index 3e9de9c2..f2d0196f 100644
--- a/karavan-app/src/main/java/org/apache/camel/karavan/api/GitResource.java
+++ b/karavan-app/src/main/java/org/apache/camel/karavan/api/ProjectGitResource.java
@@ -16,12 +16,8 @@
*/
package org.apache.camel.karavan.api;
-import org.apache.camel.karavan.model.GitRepo;
import org.apache.camel.karavan.model.Project;
-import org.apache.camel.karavan.model.ProjectFile;
-import org.apache.camel.karavan.service.GitService;
-import org.apache.camel.karavan.service.ImportService;
-import org.jboss.logging.Logger;
+import org.apache.camel.karavan.service.ProjectService;
import javax.inject.Inject;
import javax.ws.rs.Consumes;
@@ -33,24 +29,17 @@ import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import java.util.HashMap;
-
@Path("/api/git")
-public class GitResource {
-
- @Inject
- GitService gitService;
+public class ProjectGitResource {
@Inject
- ImportService importService;
-
- private static final Logger LOGGER = Logger.getLogger(GitResource.class.getName());
-
+ ProjectService projectService;
@POST
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public Project push(HashMap<String, String> params) throws Exception {
- return gitService.commitAndPushProject(params.get("projectId"), params.get("message"));
+ return projectService.commitAndPushProject(params.get("projectId"), params.get("message"));
}
@GET
@@ -58,6 +47,6 @@ public class GitResource {
@Consumes(MediaType.APPLICATION_JSON)
@Path("/{projectId}")
public Project pull(@PathParam("projectId") String projectId) throws Exception {
- return importService.importProject(projectId);
+ return projectService.importProject(projectId);
}
}
\ No newline at end of file
diff --git a/karavan-app/src/main/java/org/apache/camel/karavan/model/CommitInfo.java b/karavan-app/src/main/java/org/apache/camel/karavan/model/CommitInfo.java
new file mode 100644
index 00000000..2affc488
--- /dev/null
+++ b/karavan-app/src/main/java/org/apache/camel/karavan/model/CommitInfo.java
@@ -0,0 +1,39 @@
+package org.apache.camel.karavan.model;
+
+import java.util.List;
+
+public class CommitInfo {
+ private String commitId;
+ private Integer time;
+ private List<GitRepo> repos;
+
+ public CommitInfo(String commitId, Integer time, List<GitRepo> repos) {
+ this.commitId = commitId;
+ this.time = time;
+ this.repos = repos;
+ }
+
+ public String getCommitId() {
+ return commitId;
+ }
+
+ public void setCommitId(String commitId) {
+ this.commitId = commitId;
+ }
+
+ public Integer getTime() {
+ return time;
+ }
+
+ public void setTime(Integer time) {
+ this.time = time;
+ }
+
+ public List<GitRepo> getRepos() {
+ return repos;
+ }
+
+ public void setRepos(List<GitRepo> repos) {
+ this.repos = repos;
+ }
+}
diff --git a/karavan-app/src/main/java/org/apache/camel/karavan/service/GitService.java b/karavan-app/src/main/java/org/apache/camel/karavan/service/GitService.java
index 57765e4d..2b459861 100644
--- a/karavan-app/src/main/java/org/apache/camel/karavan/service/GitService.java
+++ b/karavan-app/src/main/java/org/apache/camel/karavan/service/GitService.java
@@ -17,27 +17,35 @@
package org.apache.camel.karavan.service;
import io.fabric8.kubernetes.api.model.Secret;
-import io.quarkus.runtime.StartupEvent;
import io.smallrye.mutiny.tuples.Tuple2;
import io.vertx.core.Vertx;
+import org.apache.camel.karavan.model.CommitInfo;
import org.apache.camel.karavan.model.GitConfig;
import org.apache.camel.karavan.model.GitRepo;
import org.apache.camel.karavan.model.GitRepoFile;
import org.apache.camel.karavan.model.Project;
import org.apache.camel.karavan.model.ProjectFile;
-import org.eclipse.jgit.api.*;
+import org.eclipse.jgit.api.CheckoutCommand;
+import org.eclipse.jgit.api.CloneCommand;
+import org.eclipse.jgit.api.FetchCommand;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.RemoteAddCommand;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.RefNotFoundException;
+import org.eclipse.jgit.diff.DiffEntry;
+import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.transport.CredentialsProvider;
+import org.eclipse.jgit.transport.FetchResult;
import org.eclipse.jgit.transport.PushResult;
import org.eclipse.jgit.transport.URIish;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
+import org.eclipse.jgit.treewalk.TreeWalk;
+import org.eclipse.jgit.treewalk.filter.TreeFilter;
import org.eclipse.microprofile.config.ConfigProvider;
import org.jboss.logging.Logger;
import javax.enterprise.context.ApplicationScoped;
-import javax.enterprise.event.Observes;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
@@ -46,16 +54,19 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.time.Instant;
-import java.time.LocalDateTime;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Base64;
+import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
@ApplicationScoped
public class GitService {
@@ -66,25 +77,69 @@ public class GitService {
@Inject
KubernetesService kubernetesService;
- @Inject
- InfinispanService infinispanService;
+ private Git git;
private static final Logger LOGGER = Logger.getLogger(GitService.class.getName());
- void onStart(@Observes StartupEvent ev) {
- LOGGER.info("Git service for repo: " + getGitConfig().getUri());
+ public Git getPollGit(){
+ if (git == null) {
+ try {
+ git = getGit(true, vertx.fileSystem().createTempDirectoryBlocking("poll"));
+ } catch (Exception e) {
+ LOGGER.error("Error", e);
+ }
+ }
+ return git;
+ }
+
+ public Map<String, Integer> getAllCommits() {
+ Map<String, Integer> result = new HashMap();
+ try {
+ Git pollGit = getPollGit();
+ if (pollGit != null) {
+ StreamSupport.stream(pollGit.log().all().call().spliterator(), false)
+ .sorted(Comparator.comparingInt(RevCommit::getCommitTime))
+ .forEach(commit -> result.put(commit.getName(), commit.getCommitTime()));
+ }
+ } catch (Exception e) {
+ LOGGER.error(e.getMessage());
+ }
+ return result;
+ }
+
+ public List<CommitInfo> getCommitsAfterCommit(int commitTime) {
+ List<CommitInfo> result = new ArrayList<>();
+ try {
+ Git pollGit = getPollGit();
+ if (pollGit != null) {
+ GitConfig gitConfig = getGitConfig();
+ CredentialsProvider cred = new UsernamePasswordCredentialsProvider(gitConfig.getUsername(), gitConfig.getPassword());
+ fetch(pollGit, cred);
+ List<RevCommit> commits = StreamSupport.stream(pollGit.log().all().call().spliterator(), false)
+ .filter(commit -> commit.getCommitTime() > commitTime)
+ .sorted(Comparator.comparingInt(RevCommit::getCommitTime)).collect(Collectors.toList());
+ for (RevCommit commit: commits) {
+ List<String> projects = new ArrayList<>(getChangedProjects(commit));
+ List<GitRepo> repo = readProjectsFromRepository(pollGit, projects.toArray(new String[projects.size()]));
+ result.add(new CommitInfo(commit.getName(), commit.getCommitTime(), repo));
+ }
+ }
+ } catch (Exception e) {
+ LOGGER.error(e.getMessage());
+ }
+ return result;
}
- private GitConfig getGitConfig() {
+ public GitConfig getGitConfig() {
String propertiesPrefix = "karavan.";
String branch = ConfigProvider.getConfig().getValue(propertiesPrefix + "git-branch", String.class);
- if (kubernetesService.inKubernetes()){
+ if (kubernetesService.inKubernetes()) {
LOGGER.info("inKubernetes " + kubernetesService.getNamespace());
- Secret secret = kubernetesService.getKaravanSecret();
+ Secret secret = kubernetesService.getKaravanSecret();
String uri = new String(Base64.getDecoder().decode(secret.getData().get("git-repository").getBytes(StandardCharsets.UTF_8)));
String username = new String(Base64.getDecoder().decode(secret.getData().get("git-username").getBytes(StandardCharsets.UTF_8)));
String password = new String(Base64.getDecoder().decode(secret.getData().get("git-password").getBytes(StandardCharsets.UTF_8)));
- if (secret.getData().containsKey("git-branch")){
+ if (secret.getData().containsKey("git-branch")) {
branch = new String(Base64.getDecoder().decode(secret.getData().get("git-branch").getBytes(StandardCharsets.UTF_8)));
}
return new GitConfig(uri, username, password, branch);
@@ -96,18 +151,6 @@ public class GitService {
}
}
- public Project commitAndPushProject(String projectId, String message) throws Exception {
- Project p = infinispanService.getProject(projectId);
- List<ProjectFile> files = infinispanService.getProjectFiles(projectId);
- RevCommit commit = commitAndPushProject(p, files, message);
- String commitId = commit.getId().getName();
- Long lastUpdate = commit.getCommitTime() * 1000L;
- p.setLastCommit(commitId);
- p.setLastCommitTimestamp(lastUpdate);
- infinispanService.saveProject(p, false);
- return p;
- }
-
public RevCommit commitAndPushProject(Project project, List<ProjectFile> files, String message) throws GitAPIException, IOException, URISyntaxException {
LOGGER.info("Commit and push project " + project.getProjectId());
GitConfig gitConfig = getGitConfig();
@@ -131,29 +174,35 @@ public class GitService {
}
public List<GitRepo> readProjectsFromRepository() {
- return readProjectsFromRepository(null);
+ Git git = null;
+ try {
+ git = getGit(true, vertx.fileSystem().createTempDirectoryBlocking(UUID.randomUUID().toString()));
+ } catch (Exception e) {
+ LOGGER.error("Error", e);
+ }
+ return readProjectsFromRepository(git, null);
}
public GitRepo readProjectFromRepository(String projectId) {
- return readProjectsFromRepository(projectId).get(0);
+ Git git = null;
+ try {
+ git = getGit(true, vertx.fileSystem().createTempDirectoryBlocking(UUID.randomUUID().toString()));
+ } catch (Exception e) {
+ LOGGER.error("Error", e);
+ }
+ return readProjectsFromRepository(git, projectId).get(0);
}
- private List<GitRepo> readProjectsFromRepository(String filter) {
+ private List<GitRepo> readProjectsFromRepository(Git git, String... filter) {
LOGGER.info("Read projects...");
- GitConfig gitConfig = getGitConfig();
- LOGGER.info("Read projects from repository " + gitConfig.getUri());
- CredentialsProvider cred = new UsernamePasswordCredentialsProvider(gitConfig.getUsername(), gitConfig.getPassword());
- String uuid = UUID.randomUUID().toString();
- String folder = vertx.fileSystem().createTempDirectoryBlocking(uuid);
- LOGGER.infof("Temp folder created: %s", folder);
List<GitRepo> result = new ArrayList<>();
- Git git = null;
try {
- git = clone(folder, gitConfig.getUri(), gitConfig.getBranch(), cred);
- checkout(git, false, null, null, gitConfig.getBranch());
+ String folder = git.getRepository().getDirectory().getAbsolutePath().replace("/.git", "");
List<String> projects = readProjectsFromFolder(folder);
if (filter != null) {
- projects = projects.stream().filter(s -> s.equals(filter)).collect(Collectors.toList());
+ projects = projects.stream().filter(s -> Arrays.stream(filter).filter(f -> f.equals(s)).findFirst().isPresent()).collect(Collectors.toList());
+ } else {
+ projects = projects.stream().filter(s -> !s.startsWith(".")).collect(Collectors.toList()); // do not import hidden folders
}
for (String project : projects) {
Map<String, String> filesRead = readProjectFilesFromFolder(folder, project);
@@ -178,6 +227,26 @@ public class GitService {
}
}
+ public Git getGit(boolean checkout, String folder) throws GitAPIException, IOException, URISyntaxException {
+ LOGGER.info("Git checkout");
+ GitConfig gitConfig = getGitConfig();
+ CredentialsProvider cred = new UsernamePasswordCredentialsProvider(gitConfig.getUsername(), gitConfig.getPassword());
+ LOGGER.info("Temp folder created " + folder);
+ Git git = null;
+ try {
+ git = clone(folder, gitConfig.getUri(), gitConfig.getBranch(), cred);
+ if (checkout) {
+ checkout(git, false, null, null, gitConfig.getBranch());
+ }
+ } catch (RefNotFoundException e) {
+ LOGGER.error("New repository");
+ git = init(folder, gitConfig.getUri(), gitConfig.getBranch());
+ } catch (Exception e) {
+ LOGGER.error("Error", e);
+ }
+ return git;
+ }
+
private List<Tuple2<String, String>> readKameletsFromFolder(String folder) {
LOGGER.info("Read kamelets from " + folder);
List<Tuple2<String, String>> kamelets = new ArrayList<>();
@@ -198,7 +267,7 @@ public class GitService {
List<String> files = new ArrayList<>();
vertx.fileSystem().readDirBlocking(folder).forEach(f -> {
String[] filenames = f.split(File.separator);
- String folderName = filenames[filenames.length -1];
+ String folderName = filenames[filenames.length - 1];
if (!folderName.startsWith(".") && Files.isDirectory(Paths.get(f))) {
LOGGER.info("Importing project from folder " + folderName);
files.add(folderName);
@@ -208,11 +277,11 @@ public class GitService {
}
private Map<String, String> readProjectFilesFromFolder(String repoFolder, String projectFolder) {
- LOGGER.infof("Read files from %s/%s", repoFolder, projectFolder );
+ LOGGER.infof("Read files from %s/%s", repoFolder, projectFolder);
Map<String, String> files = new HashMap<>();
vertx.fileSystem().readDirBlocking(repoFolder + File.separator + projectFolder).forEach(f -> {
String[] filenames = f.split(File.separator);
- String filename = filenames[filenames.length -1];
+ String filename = filenames[filenames.length - 1];
Path path = Paths.get(f);
if (!filename.startsWith(".") && !Files.isDirectory(path)) {
LOGGER.info("Importing file " + filename);
@@ -244,9 +313,9 @@ public class GitService {
LOGGER.info("Add deleted files to git index for project " + project.getProjectId());
vertx.fileSystem().readDirBlocking(path.toString()).forEach(f -> {
String[] filenames = f.split(File.separator);
- String filename = filenames[filenames.length -1];
+ String filename = filenames[filenames.length - 1];
LOGGER.info("Checking file " + filename);
- if (files.stream().filter(pf -> Objects.equals(pf.getName(), filename)).count() == 0){
+ if (files.stream().filter(pf -> Objects.equals(pf.getName(), filename)).count() == 0) {
try {
LOGGER.info("Add deleted file " + filename);
git.rm().addFilepattern(project.getProjectId() + File.separator + filename).call();
@@ -257,7 +326,7 @@ public class GitService {
});
}
- public RevCommit commitAddedAndPush(Git git, String branch, CredentialsProvider cred, String message) throws GitAPIException, IOException, URISyntaxException {
+ public RevCommit commitAddedAndPush(Git git, String branch, CredentialsProvider cred, String message) throws GitAPIException {
LOGGER.info("Commit and push changes");
LOGGER.info("Git add: " + git.add().addFilepattern(".").call());
RevCommit commit = git.commit().setMessage(message).call();
@@ -273,16 +342,16 @@ public class GitService {
return git;
}
- private void addDeletedFolderToIndex(Git git, String folder, String projectId, List<ProjectFile> files) throws IOException {
+ private void addDeletedFolderToIndex(Git git, String folder, String projectId, List<ProjectFile> files) {
LOGGER.infof("Add folder %s to git index.", projectId);
try {
- git.rm().addFilepattern(projectId + File.separator).call();
+ git.rm().addFilepattern(projectId + File.separator).call();
} catch (GitAPIException e) {
- throw new RuntimeException(e);
+ throw new RuntimeException(e);
}
}
-
- public void deleteProject(String projectId, List<ProjectFile> files) throws GitAPIException, IOException, URISyntaxException {
+
+ public void deleteProject(String projectId, List<ProjectFile> files) {
LOGGER.info("Delete and push project " + projectId);
GitConfig gitConfig = getGitConfig();
CredentialsProvider cred = new UsernamePasswordCredentialsProvider(gitConfig.getUsername(), gitConfig.getPassword());
@@ -298,7 +367,7 @@ public class GitService {
commitAddedAndPush(git, gitConfig.getBranch(), cred, commitMessage);
LOGGER.info("Delete Temp folder " + folder);
vertx.fileSystem().deleteRecursiveBlocking(folder, true);
- LOGGER.infof("Project %s deleted from Git" , projectId);
+ LOGGER.infof("Project %s deleted from Git", projectId);
} catch (RefNotFoundException e) {
LOGGER.error("Repository not found");
} catch (Exception e) {
@@ -307,14 +376,16 @@ public class GitService {
}
}
- private Git clone(String dir, String uri, String branch, CredentialsProvider cred) throws GitAPIException {
+ private Git clone(String dir, String uri, String branch, CredentialsProvider cred) throws GitAPIException, URISyntaxException {
CloneCommand cloneCommand = Git.cloneRepository();
cloneCommand.setCloneAllBranches(false);
cloneCommand.setDirectory(Paths.get(dir).toFile());
cloneCommand.setURI(uri);
cloneCommand.setBranch(branch);
cloneCommand.setCredentialsProvider(cred);
- return cloneCommand.call();
+ Git git = cloneCommand.call();
+ addRemote(git, uri);
+ return git;
}
private void addRemote(Git git, String uri) throws URISyntaxException, GitAPIException {
@@ -325,12 +396,19 @@ public class GitService {
remoteAddCommand.call();
}
+ private void fetch(Git git, CredentialsProvider cred) throws GitAPIException {
+ // fetch:
+ FetchCommand fetchCommand = git.fetch();
+ fetchCommand.setCredentialsProvider(cred);
+ FetchResult result = fetchCommand.call();
+ }
+
private void checkout(Git git, boolean create, String path, String startPoint, String branch) throws GitAPIException {
// create branch:
CheckoutCommand checkoutCommand = git.checkout();
checkoutCommand.setName(branch);
checkoutCommand.setCreateBranch(create);
- if (startPoint != null){
+ if (startPoint != null) {
checkoutCommand.setStartPoint(startPoint);
}
if (path != null) {
@@ -346,4 +424,34 @@ public class GitService {
}
return null;
}
+
+ public Set<String> getChangedProjects(RevCommit commit) {
+ Set<String> files = new HashSet<>();
+ Git git = getPollGit();
+ if (git != null) {
+ TreeWalk walk = new TreeWalk(git.getRepository());
+ walk.setRecursive(true);
+ walk.setFilter(TreeFilter.ANY_DIFF);
+
+ ObjectId a = commit.getTree().getId();
+ RevCommit parent = commit.getParent(0);
+ ObjectId b = parent.getTree().getId();
+ try {
+ walk.reset(b, a);
+ List<DiffEntry> changes = DiffEntry.scan(walk);
+ changes.stream().forEach(de -> {
+ String path = de.getNewPath();
+ if (path != null) {
+ String[] parts = path.split(File.separator);
+ if (parts.length > 0) {
+ files.add(parts[0]);
+ }
+ }
+ });
+ } catch (IOException e) {
+ LOGGER.error("Error", e);
+ }
+ }
+ return files;
+ }
}
diff --git a/karavan-app/src/main/java/org/apache/camel/karavan/service/InfinispanService.java b/karavan-app/src/main/java/org/apache/camel/karavan/service/InfinispanService.java
index e7d49982..c85bde94 100644
--- a/karavan-app/src/main/java/org/apache/camel/karavan/service/InfinispanService.java
+++ b/karavan-app/src/main/java/org/apache/camel/karavan/service/InfinispanService.java
@@ -16,6 +16,7 @@
*/
package org.apache.camel.karavan.service;
+import io.smallrye.mutiny.tuples.Tuple2;
import org.apache.camel.karavan.model.CamelStatus;
import org.apache.camel.karavan.model.DeploymentStatus;
import org.apache.camel.karavan.model.Environment;
@@ -59,6 +60,7 @@ public class InfinispanService {
BasicCache<GroupedKey, CamelStatus> camelStatuses;
BasicCache<GroupedKey, ServiceStatus> serviceStatuses;
BasicCache<String, Environment> environments;
+ BasicCache<String, String> commits;
@Inject
RemoteCacheManager cacheManager;
@@ -95,7 +97,7 @@ public class InfinispanService {
podStatuses = cacheManager.administration().withFlags(CacheContainerAdmin.AdminFlag.VOLATILE).getOrCreateCache(PodStatus.CACHE, builder.build());
serviceStatuses = cacheManager.administration().withFlags(CacheContainerAdmin.AdminFlag.VOLATILE).getOrCreateCache(ServiceStatus.CACHE, builder.build());
camelStatuses = cacheManager.administration().withFlags(CacheContainerAdmin.AdminFlag.VOLATILE).getOrCreateCache(CamelStatus.CACHE, builder.build());
-
+ commits = cacheManager.administration().withFlags(CacheContainerAdmin.AdminFlag.VOLATILE).getOrCreateCache("commits", builder.build());
cleanData();
} else {
LOGGER.info("InfinispanService is starting in remote mode");
@@ -107,6 +109,7 @@ public class InfinispanService {
podStatuses = cacheManager.administration().getOrCreateCache(PodStatus.CACHE, new XMLStringConfiguration(String.format(CACHE_CONFIG, PodStatus.CACHE)));
serviceStatuses = cacheManager.administration().getOrCreateCache(ServiceStatus.CACHE, new XMLStringConfiguration(String.format(CACHE_CONFIG, ServiceStatus.CACHE)));
camelStatuses = cacheManager.administration().getOrCreateCache(CamelStatus.CACHE, new XMLStringConfiguration(String.format(CACHE_CONFIG, CamelStatus.CACHE)));
+ commits = cacheManager.administration().getOrCreateCache("commits", new XMLStringConfiguration(String.format(CACHE_CONFIG, "commits")));
}
}
@@ -291,6 +294,23 @@ public class InfinispanService {
environments.put(environment.getName(), environment);
}
+ public void saveCommit(String commitId, int time) {
+ commits.put(commitId, String.valueOf(time));
+ }
+
+ public void saveLastCommit(String commitId) {
+ commits.put("lastCommitId", commitId);
+ }
+
+ public Tuple2<String, Integer> getLastCommit() {
+ String lastCommitId = commits.get("lastCommitId");
+ String time = commits.get(lastCommitId);
+ return Tuple2.of(lastCommitId, Integer.parseInt(time));
+ }
+
+ public boolean hasCommit(String commitId) {
+ return commits.get(commitId) != null;
+ }
protected void clearAllStatuses() {
CompletableFuture.allOf(
diff --git a/karavan-app/src/main/java/org/apache/camel/karavan/service/KaravanService.java b/karavan-app/src/main/java/org/apache/camel/karavan/service/KaravanService.java
index d235d178..6f5d7c33 100644
--- a/karavan-app/src/main/java/org/apache/camel/karavan/service/KaravanService.java
+++ b/karavan-app/src/main/java/org/apache/camel/karavan/service/KaravanService.java
@@ -67,10 +67,11 @@ public class KaravanService {
void initialImport() {
if (infinispanService.getProjects().isEmpty()) {
LOGGER.info("No projects found in the Data Grid");
- bus.publish(ImportService.IMPORT_PROJECTS, "");
+ bus.publish(ProjectService.IMPORT_PROJECTS, "");
} else {
- bus.publish(ImportService.IMPORT_TEMPLATES, "");
+ bus.publish(ProjectService.IMPORT_TEMPLATES, "");
}
+ bus.publish(ProjectService.IMPORT_COMMITS, "");
}
void startInformers() {
diff --git a/karavan-app/src/main/java/org/apache/camel/karavan/service/KubernetesService.java b/karavan-app/src/main/java/org/apache/camel/karavan/service/KubernetesService.java
index 62175b2d..796dbfc3 100644
--- a/karavan-app/src/main/java/org/apache/camel/karavan/service/KubernetesService.java
+++ b/karavan-app/src/main/java/org/apache/camel/karavan/service/KubernetesService.java
@@ -137,8 +137,8 @@ public class KubernetesService {
return "karavan-pipeline-" + environment + "-" + project.getRuntime();
}
- public String createPipelineRun(Project project) throws Exception {
- String pipeline = getPipelineName(project);
+ public String createPipelineRun(Project project) {
+ String pipeline = getPipelineName(project);
LOGGER.info("Pipeline " + pipeline + " is creating for " + project.getProjectId());
Map<String, String> labels = Map.of(
diff --git a/karavan-app/src/main/java/org/apache/camel/karavan/service/ImportService.java b/karavan-app/src/main/java/org/apache/camel/karavan/service/ProjectService.java
similarity index 60%
rename from karavan-app/src/main/java/org/apache/camel/karavan/service/ImportService.java
rename to karavan-app/src/main/java/org/apache/camel/karavan/service/ProjectService.java
index 3ebc09c4..9d8f174b 100644
--- a/karavan-app/src/main/java/org/apache/camel/karavan/service/ImportService.java
+++ b/karavan-app/src/main/java/org/apache/camel/karavan/service/ProjectService.java
@@ -16,31 +16,36 @@
*/
package org.apache.camel.karavan.service;
+import io.quarkus.scheduler.Scheduled;
import io.quarkus.vertx.ConsumeEvent;
+import io.smallrye.mutiny.tuples.Tuple2;
import org.apache.camel.karavan.model.GitRepo;
-import org.apache.camel.karavan.model.GitRepoFile;
import org.apache.camel.karavan.model.Project;
import org.apache.camel.karavan.model.ProjectFile;
+import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import java.time.Instant;
-import java.util.Arrays;
import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
@ApplicationScoped
-public class ImportService {
+public class ProjectService {
- private static final Logger LOGGER = Logger.getLogger(ImportService.class.getName());
+ private static final Logger LOGGER = Logger.getLogger(ProjectService.class.getName());
public static final String IMPORT_TEMPLATES = "import-templates";
public static final String IMPORT_PROJECTS = "import-projects";
-
+ public static final String IMPORT_COMMITS = "import-commits";
@Inject
InfinispanService infinispanService;
+ @Inject
+ KubernetesService kubernetesService;
+
@Inject
GitService gitService;
@@ -50,8 +55,37 @@ public class ImportService {
@ConfigProperty(name = "karavan.default-runtime")
String runtime;
+ private AtomicBoolean readyToPull = new AtomicBoolean(false);
+
+ @Scheduled(every = "{karavan.git-pull-interval}", concurrentExecution = Scheduled.ConcurrentExecution.SKIP)
+ void pullCommits() {
+ if (readyToPull.get()) {
+ Tuple2<String, Integer> lastCommit = infinispanService.getLastCommit();
+ gitService.getCommitsAfterCommit(lastCommit.getItem2()).forEach(commitInfo -> {
+ if (!infinispanService.hasCommit(commitInfo.getCommitId())) {
+ commitInfo.getRepos().forEach(repo -> {
+ Project project = importProjectFromRepo(repo);
+ kubernetesService.createPipelineRun(project);
+ });
+ infinispanService.saveCommit(commitInfo.getCommitId(), commitInfo.getTime());
+ }
+ infinispanService.saveLastCommit(commitInfo.getCommitId());
+ });
+ }
+ }
+
+ @ConsumeEvent(value = IMPORT_COMMITS, blocking = true)
+ void importCommits(String data) {
+ LOGGER.info("Import commits");
+ gitService.getAllCommits().forEach((commitId, time) -> {
+ infinispanService.saveCommit(commitId, time);
+ infinispanService.saveLastCommit(commitId);
+ });
+ readyToPull.set(true);
+ }
+
@ConsumeEvent(value = IMPORT_PROJECTS, blocking = true)
- void importProjects(String data) {
+ void importAllProjects(String data) {
LOGGER.info("Import projects from Git");
try {
List<GitRepo> repos = gitService.readProjectsFromRepository();
@@ -63,11 +97,7 @@ public class ImportService {
} else if (folderName.equals(Project.NAME_KAMELETS)){
project = new Project(Project.NAME_KAMELETS, "Custom Kamelets", "Custom Kamelets", "quarkus", repo.getCommitId(), repo.getLastCommitTimestamp());
} else {
- String propertiesFile = getPropertiesFile(repo);
- String projectName = getProjectName(propertiesFile);
- String projectDescription = getProjectDescription(propertiesFile);
- String runtime = getProjectRuntime(propertiesFile);
- project = new Project(folderName, projectName, projectDescription, runtime, repo.getCommitId(), repo.getLastCommitTimestamp());
+ project = getProjectFromRepo(repo);
}
infinispanService.saveProject(project, true);
@@ -87,16 +117,20 @@ public class ImportService {
LOGGER.info("Import project from Git " + projectId);
try {
GitRepo repo = gitService.readProjectFromRepository(projectId);
- Project project;
- String folderName = repo.getName();
- String propertiesFile = getPropertiesFile(repo);
- String projectName = getProjectName(propertiesFile);
- String projectDescription = getProjectDescription(propertiesFile);
- String runtime = getProjectRuntime(propertiesFile);
- project = new Project(folderName, projectName, projectDescription, runtime, repo.getCommitId(), repo.getLastCommitTimestamp());
+ return importProjectFromRepo(repo);
+ } catch (Exception e) {
+ LOGGER.error("Error during project import", e);
+ return null;
+ }
+ }
+
+ private Project importProjectFromRepo(GitRepo repo) {
+ LOGGER.info("Import project from GitRepo " + repo.getName());
+ try {
+ Project project = getProjectFromRepo(repo);
infinispanService.saveProject(project, true);
repo.getFiles().forEach(repoFile -> {
- ProjectFile file = new ProjectFile(repoFile.getName(), repoFile.getBody(), folderName, repoFile.getLastCommitTimestamp());
+ ProjectFile file = new ProjectFile(repoFile.getName(), repoFile.getBody(), repo.getName(), repoFile.getLastCommitTimestamp());
infinispanService.saveProjectFile(file);
});
return project;
@@ -106,6 +140,28 @@ public class ImportService {
}
}
+ public Project getProjectFromRepo(GitRepo repo) {
+ String folderName = repo.getName();
+ String propertiesFile = ServiceUtil.getPropertiesFile(repo);
+ String projectName = ServiceUtil.getProjectName(propertiesFile);
+ String projectDescription = ServiceUtil.getProjectDescription(propertiesFile);
+ String runtime = ServiceUtil.getProjectRuntime(propertiesFile);
+ return new Project(folderName, projectName, projectDescription, runtime, repo.getCommitId(), repo.getLastCommitTimestamp());
+ }
+
+ public Project commitAndPushProject(String projectId, String message) throws Exception {
+ Project p = infinispanService.getProject(projectId);
+ List<ProjectFile> files = infinispanService.getProjectFiles(projectId);
+ RevCommit commit = gitService.commitAndPushProject(p, files, message);
+ String commitId = commit.getId().getName();
+ Long lastUpdate = commit.getCommitTime() * 1000L;
+ p.setLastCommit(commitId);
+ p.setLastCommitTimestamp(lastUpdate);
+ infinispanService.saveProject(p, false);
+ infinispanService.saveCommit(commitId, commit.getCommitTime());
+ return p;
+ }
+
void addKameletsProject(String data) {
LOGGER.info("Add custom kamelets project if not exists");
try {
@@ -113,7 +169,7 @@ public class ImportService {
if (kamelets == null) {
kamelets = new Project(Project.NAME_KAMELETS, "Custom Kamelets", "Custom Kamelets", "quarkus", "", Instant.now().toEpochMilli());
infinispanService.saveProject(kamelets, true);
- gitService.commitAndPushProject("kamelets", "Add custom kamelets");
+ commitAndPushProject("kamelets", "Add custom kamelets");
}
} catch (Exception e) {
LOGGER.error("Error during custom kamelets project creation", e);
@@ -133,51 +189,10 @@ public class ImportService {
ProjectFile file = new ProjectFile(name, value, Project.NAME_TEMPLATES, Instant.now().toEpochMilli());
infinispanService.saveProjectFile(file);
});
- gitService.commitAndPushProject("templates", "Add default templates");
+ commitAndPushProject("templates", "Add default templates");
}
} catch (Exception e) {
LOGGER.error("Error during templates project creation", e);
}
}
-
- private String getPropertiesFile(GitRepo repo) {
- try {
- for (GitRepoFile e : repo.getFiles()){
- if (e.getName().equalsIgnoreCase("application.properties")) {
- return e.getBody();
- }
- }
- } catch (Exception e) {
- LOGGER.error(e.getMessage());
- }
- return null;
- }
-
- private static String capitalize(String str) {
- if(str == null || str.isEmpty()) {
- return str;
- }
- return str.substring(0, 1).toUpperCase() + str.substring(1);
- }
-
- private static String getProperty(String file, String property) {
- String prefix = property + "=";
- return Arrays.stream(file.split(System.lineSeparator())).filter(s -> s.startsWith(prefix))
- .findFirst().orElseGet(() -> "")
- .replace(prefix, "");
- }
-
- private static String getProjectDescription(String file) {
- String description = getProperty(file, "camel.jbang.project-description");
- return description != null && !description.isBlank() ? description : getProperty(file, "camel.karavan.project-description");
- }
-
- private static String getProjectName(String file) {
- String name = getProperty(file, "camel.jbang.project-name");
- return name != null && !name.isBlank() ? name : getProperty(file, "camel.karavan.project-name");
- }
-
- private static String getProjectRuntime(String file) {
- return getProperty(file, "camel.jbang.runtime");
- }
}
diff --git a/karavan-app/src/main/java/org/apache/camel/karavan/service/ServiceUtil.java b/karavan-app/src/main/java/org/apache/camel/karavan/service/ServiceUtil.java
new file mode 100644
index 00000000..d6d95c40
--- /dev/null
+++ b/karavan-app/src/main/java/org/apache/camel/karavan/service/ServiceUtil.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.karavan.service;
+
+import org.apache.camel.karavan.model.GitRepo;
+import org.apache.camel.karavan.model.GitRepoFile;
+
+import java.util.Arrays;
+
+public class ServiceUtil {
+
+ public static String getPropertiesFile(GitRepo repo) {
+ try {
+ for (GitRepoFile e : repo.getFiles()){
+ if (e.getName().equalsIgnoreCase("application.properties")) {
+ return e.getBody();
+ }
+ }
+ } catch (Exception e) {
+
+ }
+ return null;
+ }
+
+ public static String capitalize(String str) {
+ if(str == null || str.isEmpty()) {
+ return str;
+ }
+ return str.substring(0, 1).toUpperCase() + str.substring(1);
+ }
+
+ public static String getProperty(String file, String property) {
+ String prefix = property + "=";
+ return Arrays.stream(file.split(System.lineSeparator())).filter(s -> s.startsWith(prefix))
+ .findFirst().orElseGet(() -> "")
+ .replace(prefix, "");
+ }
+
+ public static String getProjectDescription(String file) {
+ String description = getProperty(file, "camel.jbang.project-description");
+ return description != null && !description.isBlank() ? description : getProperty(file, "camel.karavan.project-description");
+ }
+
+ public static String getProjectName(String file) {
+ String name = getProperty(file, "camel.jbang.project-name");
+ return name != null && !name.isBlank() ? name : getProperty(file, "camel.karavan.project-name");
+ }
+
+ public static String getProjectRuntime(String file) {
+ return getProperty(file, "camel.jbang.runtime");
+ }
+}
diff --git a/karavan-app/src/main/resources/application.properties b/karavan-app/src/main/resources/application.properties
index 8a27af6a..3c5236aa 100644
--- a/karavan-app/src/main/resources/application.properties
+++ b/karavan-app/src/main/resources/application.properties
@@ -9,6 +9,8 @@ karavan.git-repository=${GIT_REPOSITORY}
karavan.git-username=${GIT_USERNAME}
karavan.git-password=${GIT_TOKEN}
karavan.git-branch=main
+karavan.git-pull-interval=5s
+quarkus.scheduler.enabled=true
# Infinispan Server address
#quarkus.infinispan-client.server-list=localhost:12345
@@ -16,8 +18,8 @@ quarkus.infinispan-client.devservices.enabled=false
quarkus.infinispan-client.devservices.port=12345
quarkus.infinispan-client.devservices.service-name=karavan
# Authentication
-quarkus.infinispan-client.auth-username=admin
-quarkus.infinispan-client.auth-password=password
+quarkus.infinispan-client.username=admin
+quarkus.infinispan-client.password=password
# Infinispan client intelligence
# Use BASIC as a Docker for Mac workaround
diff --git a/karavan-app/src/main/webui/src/api/KaravanApi.tsx b/karavan-app/src/main/webui/src/api/KaravanApi.tsx
index 53af69f5..d16e1083 100644
--- a/karavan-app/src/main/webui/src/api/KaravanApi.tsx
+++ b/karavan-app/src/main/webui/src/api/KaravanApi.tsx
@@ -274,7 +274,6 @@ export class KaravanApi {
}
static async push(params: {}, after: (res: AxiosResponse<any>) => void) {
- console.log(params)
instance.post('/api/git', params)
.then(res => {
after(res);
@@ -283,6 +282,17 @@ export class KaravanApi {
});
}
+ static async pull(projectId: string, after: (res: AxiosResponse<any>) => void) {
+ instance.get('/api/git/' + projectId)
+ .then(res => {
+ if (res.status === 200) {
+ after(res.data);
+ }
+ }).catch(err => {
+ console.log(err);
+ });
+ }
+
static async getTemplatesFiles( after: (files: []) => void) {
instance.get('/api/file/templates')
.then(res => {
diff --git a/karavan-app/src/main/webui/src/projects/ProjectPageToolbar.tsx b/karavan-app/src/main/webui/src/projects/ProjectPageToolbar.tsx
index 35facb46..dc38966b 100644
--- a/karavan-app/src/main/webui/src/projects/ProjectPageToolbar.tsx
+++ b/karavan-app/src/main/webui/src/projects/ProjectPageToolbar.tsx
@@ -7,10 +7,10 @@ import {
FlexItem,
ToggleGroup,
ToggleGroupItem,
- Checkbox, Tooltip, ToolbarItem, Modal, ModalVariant, Form, FormGroup, TextInput, FormHelperText, HelperText, HelperTextItem
+ Checkbox, Tooltip, ToolbarItem, Modal, ModalVariant, Form, FormGroup, TextInput, FormHelperText
} from '@patternfly/react-core';
import '../designer/karavan.css';
-import {Project, ProjectFile, ProjectFileTypes} from "./ProjectModels";
+import {Project, ProjectFile} from "./ProjectModels";
import UploadIcon from "@patternfly/react-icons/dist/esm/icons/upload-icon";
import DownloadIcon from "@patternfly/react-icons/dist/esm/icons/download-icon";
import DownloadImageIcon from "@patternfly/react-icons/dist/esm/icons/image-icon";
@@ -18,7 +18,6 @@ import PlusIcon from "@patternfly/react-icons/dist/esm/icons/plus-icon";
import {CamelDefinitionYaml} from "karavan-core/lib/api/CamelDefinitionYaml";
import PushIcon from "@patternfly/react-icons/dist/esm/icons/code-branch-icon";
import {KaravanApi} from "../api/KaravanApi";
-import {CamelUi} from "../designer/utils/CamelUi";
interface Props {
project: Project,
@@ -164,7 +163,7 @@ export class ProjectPageToolbar extends React.Component<Props> {
commitMessageIsOpen: true,
commitMessage : commitMessage === '' ? new Date().toLocaleString() : commitMessage
})}>
- {isPushing ? "..." : "Commit"}
+ {isPushing ? "..." : "Push"}
</Button>
</Tooltip>
</FlexItem>}