You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/01/17 18:26:56 UTC
[ignite-teamcity-bot] branch ignite-10942 created (now 72d0813)
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a change to branch ignite-10942
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git.
at 72d0813 IGNITE-10942:: Optimize background JIRA tickets sync
This branch includes the following new commits:
new 72d0813 IGNITE-10942:: Optimize background JIRA tickets sync
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[ignite-teamcity-bot] 01/01: IGNITE-10942:: Optimize background
JIRA tickets sync
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-10942
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 72d0813bacb1d7eec3920791982a1bddf3ebb195
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Jan 17 21:26:51 2019 +0300
IGNITE-10942:: Optimize background JIRA tickets sync
---
.../ci/github/ignited/GitHubConnIgnitedImpl.java | 11 ++---
.../ignite/ci/jira/ignited/JiraTicketDao.java | 25 ++++++++---
.../ignite/ci/jira/ignited/JiraTicketSync.java | 51 +++++++++++++++-------
.../ignite/ci/jira/ignited/TicketCompacted.java | 4 +-
.../ignite/ci/jira/pure/IJiraIntegration.java | 3 ++
.../java/org/apache/ignite/ci/jira/pure/Jira.java | 9 ++--
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 2 +-
7 files changed, 68 insertions(+), 37 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
index 2838936..3cbe059 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
@@ -110,14 +110,9 @@ class GitHubConnIgnitedImpl implements IGitHubConnIgnited {
runActualizePrs(srvId, false);
// schedule full resync later
- scheduler.invokeLater(this::sheduleResync, 20, TimeUnit.SECONDS);
- }
-
- /**
- *
- */
- private void sheduleResync() {
- scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 60, TimeUnit.MINUTES);
+ scheduler.invokeLater(
+ () -> scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 2, TimeUnit.HOURS),
+ 1, TimeUnit.MINUTES);
}
/**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java
index 6502488..30def78 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ci.jira.ignited;
import com.google.common.base.Preconditions;
import java.util.Collection;
+import java.util.Map;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
@@ -89,19 +90,19 @@ public class JiraTicketDao {
/**
* Save small part of loaded mutes.
- *
* @param srvIdMaskHigh Server id mask high.
* @param chunk Chunk.
* @param ticketPrefix Ticket name template.
+ * @return number of tickets totally saved.
*/
@AutoProfiling
- public void saveChunk(int srvIdMaskHigh, Collection<Ticket> chunk, String ticketPrefix) {
+ public int saveChunk(int srvIdMaskHigh, Collection<Ticket> chunk, String ticketPrefix) {
Preconditions.checkNotNull(jiraCache, "init() was not called");
if (F.isEmpty(chunk))
- return;
+ return 0;
- HashMap<Long, TicketCompacted> compactedTickets = new HashMap<>(U.capacity(chunk.size()));
+ Map<Long, TicketCompacted> compactedTickets = new HashMap<>(U.capacity(chunk.size()));
for (Ticket ticket : chunk) {
long key = ticketToCacheKey(srvIdMaskHigh, ticket.igniteId(ticketPrefix));
@@ -110,6 +111,20 @@ public class JiraTicketDao {
compactedTickets.put(key, val);
}
- jiraCache.putAll(compactedTickets);
+ Map<Long, TicketCompacted> dbVal = jiraCache.getAll(compactedTickets.keySet());
+
+ Map<Long, TicketCompacted> ticketsToUpdate = new HashMap<>(U.capacity(chunk.size()));
+
+ compactedTickets.forEach((k, v) -> {
+ TicketCompacted existing = dbVal.get(k);
+
+ if (existing == null || !v.equals(existing))
+ ticketsToUpdate.put(k, v);
+ });
+
+ if(!ticketsToUpdate.isEmpty())
+ jiraCache.putAll(ticketsToUpdate);
+
+ return ticketsToUpdate.size();
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java
index 178ced6..598f074 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java
@@ -49,8 +49,23 @@ public class JiraTicketSync {
* @param srvId Server ID
*/
public void ensureActualizeJiraTickets(String srvId) {
- scheduler.sheduleNamed(taskName("actualizeJiraTickets", srvId),
- () -> actualizeJiraTickets(srvId), 15, TimeUnit.MINUTES);
+ scheduler.sheduleNamed(taskName("incrementalSync", srvId),
+ () -> incrementalUpdate(srvId), 15, TimeUnit.MINUTES);
+ }
+
+ /**
+ * @param srvId Server id.
+ */
+ public String incrementalUpdate(String srvId) {
+ String res = actualizeJiraTickets(srvId, false);
+
+ scheduler.invokeLater(() -> {
+ scheduler.sheduleNamed(taskName("fullResync", srvId),
+ () -> actualizeJiraTickets(srvId, true), 2, TimeUnit.HOURS);
+ },
+ 5, TimeUnit.MINUTES);
+
+ return res;
}
/**
@@ -64,9 +79,10 @@ public class JiraTicketSync {
}
/**
* @param srvId Server internal identification.
+ * @param fullResync full or incremental.
*/
@MonitoredTask(name = "Actualize Jira", nameExtArgsIndexes = {0})
- protected String actualizeJiraTickets(String srvId) {
+ protected String actualizeJiraTickets(String srvId, boolean fullResync) {
int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
IJiraIntegration jira = jiraIntegrationProvider.server(srvId);
@@ -81,26 +97,31 @@ public class JiraTicketSync {
if (F.isEmpty(page))
return "Something went wrong - no tickets found. Check jira availability.";
- jiraDao.saveChunk(srvIdMaskHigh, page, jira.ticketPrefix());
+ int ticketsSaved = jiraDao.saveChunk(srvIdMaskHigh, page, jira.ticketPrefix());
+
+ int ticketsProcessed = page.size();
- int ticketsSaved = page.size();
+ if (ticketsSaved != 0 || fullResync) {
+ while (tickets.nextStart() > 0) {
+ url = baseUrl + "&startAt=" + tickets.nextStart();
- while (tickets.nextStart() > 0) {
- url = baseUrl + "&startAt=" + tickets.nextStart();
+ tickets = jira.getTicketsPage(srvId, url);
- tickets = jira.getTicketsPage(srvId, url);
+ page = tickets.issuesNotNull();
- page = tickets.issuesNotNull();
+ if (F.isEmpty(page))
+ break;
- if (F.isEmpty(page))
- break;
+ int savedNow = jiraDao.saveChunk(srvIdMaskHigh, page, jira.ticketPrefix());
- //todo find not updated chunk and exit
- jiraDao.saveChunk(srvIdMaskHigh, page, jira.ticketPrefix());
+ ticketsSaved += savedNow;
+ ticketsProcessed += page.size();
- ticketsSaved += page.size();
+ if (savedNow == 0 && !fullResync)
+ break; // find not updated chunk and exit
+ }
}
- return "Jira tickets saved " + ticketsSaved + " for " + srvId;
+ return "Jira tickets saved " + ticketsSaved + " from" + ticketsProcessed + " for " + srvId;
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java
index d1a22f9..4eb942e 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java
@@ -30,10 +30,10 @@ public class TicketCompacted {
/** Id. */
public long id;
- /** Ticket full name like "IGNITE-123". */
+ /** Ticket number, integer value like 123 from name like "IGNITE-123". */
public int igniteId;
- /** Fields. */
+ /** Id of string: Fields/status/name, value compacted. */
public int status;
/**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java
index 6ff8fea..ec4c782 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java
@@ -81,5 +81,8 @@ public interface IJiraIntegration {
String getJiraApiUrl();
+ /**
+ * @return {@code True} if JIRA authorization token is available.
+ */
boolean isJiraTokenAvailable();
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java
index e5a006e..3fd177e 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
import static com.google.common.base.Strings.isNullOrEmpty;
/**
- *
+ * Implementation of pure JIRA interaction.
*/
class Jira implements IJiraIntegration {
/** Logger. */
@@ -51,6 +51,8 @@ class Jira implements IJiraIntegration {
/** URL for JIRA integration. */
private String jiraApiUrl;
+
+ /** Server id. */
private String srvId;
/** {@inheritDoc} */
@@ -70,11 +72,6 @@ class Jira implements IJiraIntegration {
jiraApiUrl = props.getProperty(HelperConfig.JIRA_API_URL);
}
- /**
- * @return {@code True} if JIRA authorization token is available.
- */
- // boolean isJiraTokenAvailable();
-
/** {@inheritDoc} */
@Override public String jiraUrl() {
return jiraUrl;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 3938c27..073f1a7 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -597,7 +597,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
*
*/
private void sheduleResyncBuildRefs() {
- scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 120, TimeUnit.MINUTES);
+ scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 2, TimeUnit.HOURS);
}
/**