You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/01/18 12:02:30 UTC
[ignite-teamcity-bot] branch master updated: IGNITE-10942: Optimize
background JIRA tickets sync done - Fixes #108.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new d607072 IGNITE-10942: Optimize background JIRA tickets sync done - Fixes #108.
d607072 is described below
commit d6070723ace93648e0aa2f204c6d9fa33a412380
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Jan 18 15:01:53 2019 +0300
IGNITE-10942: Optimize background JIRA tickets sync done - Fixes #108.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
---
.../ci/github/ignited/GitHubConnIgnitedImpl.java | 11 ++---
.../ignite/ci/jira/ignited/JiraTicketDao.java | 25 ++++++++--
.../ignite/ci/jira/ignited/JiraTicketSync.java | 53 +++++++++++++++-------
.../ignite/ci/jira/ignited/TicketCompacted.java | 4 +-
.../ignite/ci/jira/pure/IJiraIntegration.java | 3 ++
.../java/org/apache/ignite/ci/jira/pure/Jira.java | 9 ++--
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 2 +-
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
8 files changed, 70 insertions(+), 39 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
index 2838936..dcbf1f4 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
@@ -110,14 +110,9 @@ class GitHubConnIgnitedImpl implements IGitHubConnIgnited {
runActualizePrs(srvId, false);
// schedule full resync later
- scheduler.invokeLater(this::sheduleResync, 20, TimeUnit.SECONDS);
- }
-
- /**
- *
- */
- private void sheduleResync() {
- scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 60, TimeUnit.MINUTES);
+ scheduler.invokeLater(
+ () -> scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 2, TimeUnit.HOURS),
+ 5, TimeUnit.MINUTES);
}
/**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java
index 6502488..30def78 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ci.jira.ignited;
import com.google.common.base.Preconditions;
import java.util.Collection;
+import java.util.Map;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
@@ -89,19 +90,19 @@ public class JiraTicketDao {
/**
* Save small part of loaded mutes.
- *
* @param srvIdMaskHigh Server id mask high.
* @param chunk Chunk.
* @param ticketPrefix Ticket name template.
+ * @return number of tickets totally saved.
*/
@AutoProfiling
- public void saveChunk(int srvIdMaskHigh, Collection<Ticket> chunk, String ticketPrefix) {
+ public int saveChunk(int srvIdMaskHigh, Collection<Ticket> chunk, String ticketPrefix) {
Preconditions.checkNotNull(jiraCache, "init() was not called");
if (F.isEmpty(chunk))
- return;
+ return 0;
- HashMap<Long, TicketCompacted> compactedTickets = new HashMap<>(U.capacity(chunk.size()));
+ Map<Long, TicketCompacted> compactedTickets = new HashMap<>(U.capacity(chunk.size()));
for (Ticket ticket : chunk) {
long key = ticketToCacheKey(srvIdMaskHigh, ticket.igniteId(ticketPrefix));
@@ -110,6 +111,20 @@ public class JiraTicketDao {
compactedTickets.put(key, val);
}
- jiraCache.putAll(compactedTickets);
+ Map<Long, TicketCompacted> dbVal = jiraCache.getAll(compactedTickets.keySet());
+
+ Map<Long, TicketCompacted> ticketsToUpdate = new HashMap<>(U.capacity(chunk.size()));
+
+ compactedTickets.forEach((k, v) -> {
+ TicketCompacted existing = dbVal.get(k);
+
+ if (existing == null || !v.equals(existing))
+ ticketsToUpdate.put(k, v);
+ });
+
+ if(!ticketsToUpdate.isEmpty())
+ jiraCache.putAll(ticketsToUpdate);
+
+ return ticketsToUpdate.size();
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java
index 178ced6..28ee60f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java
@@ -49,8 +49,23 @@ public class JiraTicketSync {
* @param srvId Server ID
*/
public void ensureActualizeJiraTickets(String srvId) {
- scheduler.sheduleNamed(taskName("actualizeJiraTickets", srvId),
- () -> actualizeJiraTickets(srvId), 15, TimeUnit.MINUTES);
+ scheduler.sheduleNamed(taskName("incrementalSync", srvId),
+ () -> incrementalUpdate(srvId), 15, TimeUnit.MINUTES);
+ }
+
+ /**
+ * @param srvId Server id.
+ */
+ public String incrementalUpdate(String srvId) {
+ String res = actualizeJiraTickets(srvId, false);
+
+ scheduler.invokeLater(() -> {
+ scheduler.sheduleNamed(taskName("fullResync", srvId),
+ () -> actualizeJiraTickets(srvId, true), 2, TimeUnit.HOURS);
+ },
+ 5, TimeUnit.MINUTES);
+
+ return res;
}
/**
@@ -64,9 +79,10 @@ public class JiraTicketSync {
}
/**
* @param srvId Server internal identification.
+ * @param fullResync full or incremental.
*/
- @MonitoredTask(name = "Actualize Jira", nameExtArgsIndexes = {0})
- protected String actualizeJiraTickets(String srvId) {
+ @MonitoredTask(name = "Actualize Jira(srv, full resync)", nameExtArgsIndexes = {0, 1})
+ protected String actualizeJiraTickets(String srvId, boolean fullResync) {
int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
IJiraIntegration jira = jiraIntegrationProvider.server(srvId);
@@ -81,26 +97,31 @@ public class JiraTicketSync {
if (F.isEmpty(page))
return "Something went wrong - no tickets found. Check jira availability.";
- jiraDao.saveChunk(srvIdMaskHigh, page, jira.ticketPrefix());
+ int ticketsSaved = jiraDao.saveChunk(srvIdMaskHigh, page, jira.ticketPrefix());
+
+ int ticketsProcessed = page.size();
- int ticketsSaved = page.size();
+ if (ticketsSaved != 0 || fullResync) {
+ while (tickets.nextStart() > 0) {
+ url = baseUrl + "&startAt=" + tickets.nextStart();
- while (tickets.nextStart() > 0) {
- url = baseUrl + "&startAt=" + tickets.nextStart();
+ tickets = jira.getTicketsPage(srvId, url);
- tickets = jira.getTicketsPage(srvId, url);
+ page = tickets.issuesNotNull();
- page = tickets.issuesNotNull();
+ if (F.isEmpty(page))
+ break;
- if (F.isEmpty(page))
- break;
+ int savedNow = jiraDao.saveChunk(srvIdMaskHigh, page, jira.ticketPrefix());
- //todo find not updated chunk and exit
- jiraDao.saveChunk(srvIdMaskHigh, page, jira.ticketPrefix());
+ ticketsSaved += savedNow;
+ ticketsProcessed += page.size();
- ticketsSaved += page.size();
+ if (savedNow == 0 && !fullResync)
+ break; // find not updated chunk and exit
+ }
}
- return "Jira tickets saved " + ticketsSaved + " for " + srvId;
+ return "Jira tickets saved " + ticketsSaved + " from " + ticketsProcessed + " checked for service " + srvId;
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java
index d1a22f9..4eb942e 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java
@@ -30,10 +30,10 @@ public class TicketCompacted {
/** Id. */
public long id;
- /** Ticket full name like "IGNITE-123". */
+ /** Ticket number, integer value like 123 from name like "IGNITE-123". */
public int igniteId;
- /** Fields. */
+ /** Id of string: Fields/status/name, value compacted. */
public int status;
/**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java
index 6ff8fea..ec4c782 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java
@@ -81,5 +81,8 @@ public interface IJiraIntegration {
String getJiraApiUrl();
+ /**
+ * @return {@code True} if JIRA authorization token is available.
+ */
boolean isJiraTokenAvailable();
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java
index e5a006e..3fd177e 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
import static com.google.common.base.Strings.isNullOrEmpty;
/**
- *
+ * Implementation of pure JIRA interaction.
*/
class Jira implements IJiraIntegration {
/** Logger. */
@@ -51,6 +51,8 @@ class Jira implements IJiraIntegration {
/** URL for JIRA integration. */
private String jiraApiUrl;
+
+ /** Server id. */
private String srvId;
/** {@inheritDoc} */
@@ -70,11 +72,6 @@ class Jira implements IJiraIntegration {
jiraApiUrl = props.getProperty(HelperConfig.JIRA_API_URL);
}
- /**
- * @return {@code True} if JIRA authorization token is available.
- */
- // boolean isJiraTokenAvailable();
-
/** {@inheritDoc} */
@Override public String jiraUrl() {
return jiraUrl;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 3938c27..073f1a7 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -597,7 +597,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
*
*/
private void sheduleResyncBuildRefs() {
- scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 120, TimeUnit.MINUTES);
+ scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 2, TimeUnit.HOURS);
}
/**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
index f3c1fb3..bb7f69a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
@@ -28,7 +28,7 @@ package org.apache.ignite.ci.web.model;
public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot";
/** TC Bot Version. */
- public static final String VERSION = "20190117";
+ public static final String VERSION = "20190118";
/** TC Bot Version. */
public String version = VERSION;