You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/11/10 10:21:46 UTC
[ignite-teamcity-bot] 02/02: Proactive build sync: protection from
loading duplicate builds
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit bc06909ff25e2d04a93ce7572058f3975a16fa60
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Sat Nov 10 13:21:38 2018 +0300
Proactive build sync: protection from loading duplicate builds
---
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 9 +--
.../ignited/fatbuild/ProactiveFatBuildSync.java | 66 +++++++++++++++++++---
2 files changed, 60 insertions(+), 15 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index ec1bf22..b6b0f04 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -330,14 +330,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
ensureActualizeRequested();
FatBuildCompacted existingBuild = fatBuildDao.getFatBuild(srvIdMaskHigh, buildId);
- if (existingBuild != null && !existingBuild.isOutdatedEntityVersion()) {
- boolean finished = !existingBuild.isRunning(compactor) && !existingBuild.isQueued(compactor);
-
- if(finished || acceptQueued)
- return existingBuild;
- }
-
- FatBuildCompacted savedVer = buildSync.reloadBuild(conn, buildId, existingBuild);
+ FatBuildCompacted savedVer = buildSync.loadBuild(conn, buildId, existingBuild, acceptQueued);
//build was modified, probably we need also to update reference accordindly
if (savedVer != null)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index 47d7480..dd111dd 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -32,6 +32,7 @@ import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
import org.apache.ignite.ci.util.ExceptionUtil;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,9 +69,18 @@ public class ProactiveFatBuildSync {
@GuardedBy("this")
private Map<String, SyncTask> buildToLoad = new HashMap<>();
+ public void doLoadBuilds(int i, String srvNme, ITeamcityConn conn, Set<Integer> paginateUntil) {
+ doLoadBuilds(i, srvNme, conn, paginateUntil, getSyncTask(conn).loadingBuilds);
+ }
+
+ /**
+ * Scope of work: builds to be loaded from a connection.
+ */
private static class SyncTask {
ITeamcityConn conn;
Set<Integer> ids = new HashSet<>();
+
+ GridConcurrentHashSet<Integer> loadingBuilds = new GridConcurrentHashSet<>();
}
/**
@@ -84,10 +94,11 @@ public class ProactiveFatBuildSync {
synchronized (this) {
- final String serverId = conn.serverId();
- final SyncTask syncTask = buildToLoad.computeIfAbsent(serverId, s -> new SyncTask());
- syncTask.conn = conn;
- syncTask.ids.addAll(buildsToAskFromTc);
+ final SyncTask syncTask = getSyncTask(conn);
+
+ buildsToAskFromTc.stream()
+ .filter(id -> !syncTask.loadingBuilds.contains(id))
+ .forEach(syncTask.ids::add);
}
int ldrToActivate = ThreadLocalRandom.current().nextInt(FAT_BUILD_PROACTIVE_TASKS);
@@ -97,6 +108,14 @@ public class ProactiveFatBuildSync {
}
+ @NotNull
+ public synchronized SyncTask getSyncTask(ITeamcityConn conn) {
+ final String serverId = conn.serverId();
+ final SyncTask syncTask = buildToLoad.computeIfAbsent(serverId, s -> new SyncTask());
+ syncTask.conn = conn;
+ return syncTask;
+ }
+
@SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
@MonitoredTask(name = "Find missing builds", nameExtArgsIndexes = {0})
@AutoProfiling
@@ -131,6 +150,8 @@ public class ProactiveFatBuildSync {
private void loadFatBuilds(int ldrNo, String serverId) {
Set<Integer> load;
ITeamcityConn conn;
+ final GridConcurrentHashSet<Integer> loadingBuilds;
+
synchronized (this) {
final SyncTask syncTask = buildToLoad.get(serverId);
if (syncTask == null)
@@ -145,19 +166,25 @@ public class ProactiveFatBuildSync {
return;
load = syncTask.ids;
+ //marking that builds are in progress
+
+ loadingBuilds = syncTask.loadingBuilds;
+ loadingBuilds.addAll(load);
+
syncTask.ids = new HashSet<>();
conn = syncTask.conn;
syncTask.conn = null;
}
- doLoadBuilds(ldrNo, serverId, conn, load);
+ doLoadBuilds(ldrNo, serverId, conn, load, loadingBuilds);
}
@SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
@MonitoredTask(name = "Proactive Builds Loading (srv,agent)", nameExtArgsIndexes = {1, 0})
@AutoProfiling
- public String doLoadBuilds(int ldrNo, String srvId, ITeamcityConn conn, Set<Integer> load) {
+ public String doLoadBuilds(int ldrNo, String srvId, ITeamcityConn conn, Set<Integer> load,
+ GridConcurrentHashSet<Integer> loadingBuilds) {
if(load.isEmpty())
return "Nothing to load";
@@ -173,10 +200,12 @@ public class ProactiveFatBuildSync {
try {
FatBuildCompacted existingBuild = builds.get(FatBuildDao.buildIdToCacheKey(srvIdMaskHigh, buildId));
- FatBuildCompacted savedVer = reloadBuild(conn, buildId, existingBuild);
+ FatBuildCompacted savedVer = loadBuild(conn, buildId, existingBuild, false);
if (savedVer != null)
ld.incrementAndGet();
+
+ loadingBuilds.remove(buildId);
}
catch (Exception e) {
logger.error("", e);
@@ -198,6 +227,29 @@ public class ProactiveFatBuildSync {
() -> findMissingBuildsFromBuildRef(srvName, conn), 360, TimeUnit.MINUTES);
}
+
+ /**
+ *
+ * @param conn
+ * @param buildId
+ * @param existingBuild
+ * @param acceptQueued
+ * @return null if nothing was saved, use existing build
+ */
+ @Nullable
+ public FatBuildCompacted loadBuild(ITeamcityConn conn, int buildId,
+ @Nullable FatBuildCompacted existingBuild,
+ boolean acceptQueued) {
+ if (existingBuild != null && !existingBuild.isOutdatedEntityVersion()) {
+ boolean finished = !existingBuild.isRunning(compactor) && !existingBuild.isQueued(compactor);
+
+ if(finished || acceptQueued)
+ return null;
+ }
+
+ return reloadBuild(conn, buildId, existingBuild);
+ }
+
/**
*
* @param conn