You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2021/04/16 11:13:50 UTC
[camel] branch master updated: CAMEL-16515: camel-github - Commit
consumer should not grow internal list of commit shas forever. Instead keep
a chunk of last, and only poll from last sha to filter duplictes.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/master by this push:
new 7efcad4 CAMEL-16515: camel-github - Commit consumer should not grow internal list of commit shas forever. Instead keep a chunk of last, and only poll from last sha to filter duplictes.
7efcad4 is described below
commit 7efcad44008154012c614bb5a4acbccc76947239
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Fri Apr 16 13:13:10 2021 +0200
CAMEL-16515: camel-github - Commit consumer should not grow internal list of commit shas forever. Instead keep a chunk of last, and only poll from last sha to filter duplictes.
---
.../component/github/consumer/CommitConsumer.java | 41 ++++++++++++++++++++--
...> CommitConsumerSkipExistingOnStartupTest.java} | 23 +++++++-----
.../github/consumer/CommitConsumerTest.java | 8 ++---
.../github/services/MockCommitService.java | 17 +++++++--
4 files changed, 69 insertions(+), 20 deletions(-)
diff --git a/components/camel-github/src/main/java/org/apache/camel/component/github/consumer/CommitConsumer.java b/components/camel-github/src/main/java/org/apache/camel/component/github/consumer/CommitConsumer.java
index 5a4bbf7..d8660f9 100644
--- a/components/camel-github/src/main/java/org/apache/camel/component/github/consumer/CommitConsumer.java
+++ b/components/camel-github/src/main/java/org/apache/camel/component/github/consumer/CommitConsumer.java
@@ -16,9 +16,10 @@
*/
package org.apache.camel.component.github.consumer;
-import java.util.ArrayList;
import java.util.List;
+import java.util.Queue;
import java.util.Stack;
+import java.util.concurrent.ArrayBlockingQueue;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
@@ -34,11 +35,15 @@ public class CommitConsumer extends AbstractGitHubConsumer {
private static final transient Logger LOG = LoggerFactory.getLogger(CommitConsumer.class);
private CommitService commitService;
+ private String branchName;
- private List<String> commitHashes = new ArrayList<>();
+ // keep a chunk of the last hashes so we can filter out duplicates
+ private Queue<String> commitHashes = new ArrayBlockingQueue<>(100);
+ private volatile String lastSha;
public CommitConsumer(GitHubEndpoint endpoint, Processor processor, String branchName) throws Exception {
super(endpoint, processor);
+ this.branchName = branchName;
Registry registry = endpoint.getCamelContext().getRegistry();
Object service = registry.lookupByName(GitHubConstants.GITHUB_COMMIT_SERVICE);
@@ -48,24 +53,54 @@ public class CommitConsumer extends AbstractGitHubConsumer {
} else {
commitService = new CommitService();
}
+ }
+
+ @Override
+ protected void doInit() throws Exception {
+ super.doInit();
initService(commitService);
+ }
+
+ @Override
+ protected void doStart() throws Exception {
+ super.doStart();
+
+ // ensure we start from clean
+ commitHashes.clear();
+ lastSha = null;
LOG.info("GitHub CommitConsumer: Indexing current commits...");
List<RepositoryCommit> commits = commitService.getCommits(getRepository(), branchName, null);
for (RepositoryCommit commit : commits) {
commitHashes.add(commit.getSha());
+ lastSha = commit.getSha();
}
}
@Override
+ protected void doStop() throws Exception {
+ super.doStop();
+
+ commitHashes.clear();
+ lastSha = null;
+ }
+
+ @Override
protected int poll() throws Exception {
- List<RepositoryCommit> commits = commitService.getCommits(getRepository());
+ List<RepositoryCommit> commits;
+ if (lastSha != null) {
+ commits = commitService.getCommits(getRepository(), lastSha, null);
+ } else {
+ commits = commitService.getCommits(getRepository());
+ }
+
// In the end, we want tags oldest to newest.
Stack<RepositoryCommit> newCommits = new Stack<>();
for (RepositoryCommit commit : commits) {
if (!commitHashes.contains(commit.getSha())) {
newCommits.push(commit);
commitHashes.add(commit.getSha());
+ lastSha = commit.getSha();
}
}
diff --git a/components/camel-github/src/test/java/org/apache/camel/component/github/consumer/CommitConsumerTest.java b/components/camel-github/src/test/java/org/apache/camel/component/github/consumer/CommitConsumerSkipExistingOnStartupTest.java
similarity index 77%
copy from components/camel-github/src/test/java/org/apache/camel/component/github/consumer/CommitConsumerTest.java
copy to components/camel-github/src/test/java/org/apache/camel/component/github/consumer/CommitConsumerSkipExistingOnStartupTest.java
index bd8be61..b44cd59 100644
--- a/components/camel-github/src/test/java/org/apache/camel/component/github/consumer/CommitConsumerTest.java
+++ b/components/camel-github/src/test/java/org/apache/camel/component/github/consumer/CommitConsumerSkipExistingOnStartupTest.java
@@ -17,15 +17,14 @@
package org.apache.camel.component.github.consumer;
import org.apache.camel.Exchange;
-import org.apache.camel.Message;
import org.apache.camel.Processor;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.component.github.GitHubComponentTestBase;
import org.apache.camel.component.github.GitHubConstants;
-import org.eclipse.egit.github.core.RepositoryCommit;
import org.junit.jupiter.api.Test;
-public class CommitConsumerTest extends GitHubComponentTestBase {
+public class CommitConsumerSkipExistingOnStartupTest extends GitHubComponentTestBase {
+
@Override
protected RouteBuilder createRouteBuilder() throws Exception {
return new RouteBuilder() {
@@ -33,6 +32,7 @@ public class CommitConsumerTest extends GitHubComponentTestBase {
@Override
public void configure() throws Exception {
from("github://commit/master?repoOwner=anotherguy&repoName=somerepo")
+ .routeId("foo").noAutoStartup()
.process(new GitHubCommitProcessor())
.to(mockResultEndpoint);
}
@@ -41,12 +41,18 @@ public class CommitConsumerTest extends GitHubComponentTestBase {
@Test
public void commitConsumerTest() throws Exception {
- mockResultEndpoint.expectedMessageCount(2);
- RepositoryCommit commit1 = commitService.addRepositoryCommit();
- RepositoryCommit commit2 = commitService.addRepositoryCommit();
- mockResultEndpoint.expectedBodiesReceivedInAnyOrder(commit1.getCommit().getMessage(), commit2.getCommit().getMessage());
+ // add 2 commits before starting route
+ commitService.addRepositoryCommit("test-1");
+ commitService.addRepositoryCommit("test-2");
+
+ mockResultEndpoint.expectedMessageCount(3);
+ mockResultEndpoint.expectedBodiesReceivedInAnyOrder("test-3", "test-4", "test-5");
+
+ context.getRouteController().startAllRoutes();
- Thread.sleep(1 * 1000);
+ commitService.addRepositoryCommit("test-3");
+ commitService.addRepositoryCommit("test-4");
+ commitService.addRepositoryCommit("test-5");
mockResultEndpoint.assertIsSatisfied();
}
@@ -54,7 +60,6 @@ public class CommitConsumerTest extends GitHubComponentTestBase {
public class GitHubCommitProcessor implements Processor {
@Override
public void process(Exchange exchange) throws Exception {
- Message in = exchange.getIn();
String author = exchange.getMessage().getHeader(GitHubConstants.GITHUB_COMMIT_AUTHOR, String.class);
String sha = exchange.getMessage().getHeader(GitHubConstants.GITHUB_COMMIT_SHA, String.class);
if (log.isDebugEnabled()) {
diff --git a/components/camel-github/src/test/java/org/apache/camel/component/github/consumer/CommitConsumerTest.java b/components/camel-github/src/test/java/org/apache/camel/component/github/consumer/CommitConsumerTest.java
index bd8be61..3cbf4cd 100644
--- a/components/camel-github/src/test/java/org/apache/camel/component/github/consumer/CommitConsumerTest.java
+++ b/components/camel-github/src/test/java/org/apache/camel/component/github/consumer/CommitConsumerTest.java
@@ -17,7 +17,6 @@
package org.apache.camel.component.github.consumer;
import org.apache.camel.Exchange;
-import org.apache.camel.Message;
import org.apache.camel.Processor;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.component.github.GitHubComponentTestBase;
@@ -42,19 +41,16 @@ public class CommitConsumerTest extends GitHubComponentTestBase {
@Test
public void commitConsumerTest() throws Exception {
mockResultEndpoint.expectedMessageCount(2);
- RepositoryCommit commit1 = commitService.addRepositoryCommit();
- RepositoryCommit commit2 = commitService.addRepositoryCommit();
+ RepositoryCommit commit1 = commitService.addRepositoryCommit("test-1");
+ RepositoryCommit commit2 = commitService.addRepositoryCommit("test-2");
mockResultEndpoint.expectedBodiesReceivedInAnyOrder(commit1.getCommit().getMessage(), commit2.getCommit().getMessage());
- Thread.sleep(1 * 1000);
-
mockResultEndpoint.assertIsSatisfied();
}
public class GitHubCommitProcessor implements Processor {
@Override
public void process(Exchange exchange) throws Exception {
- Message in = exchange.getIn();
String author = exchange.getMessage().getHeader(GitHubConstants.GITHUB_COMMIT_AUTHOR, String.class);
String sha = exchange.getMessage().getHeader(GitHubConstants.GITHUB_COMMIT_SHA, String.class);
if (log.isDebugEnabled()) {
diff --git a/components/camel-github/src/test/java/org/apache/camel/component/github/services/MockCommitService.java b/components/camel-github/src/test/java/org/apache/camel/component/github/services/MockCommitService.java
index 45d0424..f55a021 100644
--- a/components/camel-github/src/test/java/org/apache/camel/component/github/services/MockCommitService.java
+++ b/components/camel-github/src/test/java/org/apache/camel/component/github/services/MockCommitService.java
@@ -39,7 +39,7 @@ public class MockCommitService extends CommitService {
private AtomicLong fakeSha = new AtomicLong(System.currentTimeMillis());
private Map<String, CommitStatus> commitStatus = new HashMap<>();
- public synchronized RepositoryCommit addRepositoryCommit() {
+ public synchronized RepositoryCommit addRepositoryCommit(String message) {
User author = new User();
author.setEmail("someguy@gmail.com"); // TODO change
author.setHtmlUrl("http://github/someguy");
@@ -50,7 +50,11 @@ public class MockCommitService extends CommitService {
rc.setSha(fakeSha.incrementAndGet() + "");
rc.setCommitter(author);
Commit commit = new Commit();
- commit.setMessage("Test");
+ if (message == null) {
+ commit.setMessage("Test");
+ } else {
+ commit.setMessage(message);
+ }
rc.setCommit(commit);
LOG.debug("In MockCommitService added commit with sha " + rc.getSha());
commitsList.add(rc);
@@ -62,6 +66,15 @@ public class MockCommitService extends CommitService {
public synchronized List<RepositoryCommit> getCommits(IRepositoryIdProvider repository, String sha, String path)
throws IOException {
LOG.debug("Returning list of size " + commitsList.size());
+
+ if (sha != null) {
+ for (int i = 0; i < commitsList.size(); i++) {
+ RepositoryCommit commit = commitsList.get(i);
+ if (commit.getSha().equals(sha)) {
+ return commitsList.subList(i, commitsList.size());
+ }
+ }
+ }
return commitsList;
}