You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/27 11:46:05 UTC
[ignite-teamcity-bot] branch ignite-9848-2 updated: IGNITE-9848:
Standalone syncer class; Find missing builds by refs done
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-2
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9848-2 by this push:
new a5c34f4 IGNITE-9848: Standalone syncer class; Find missing builds by refs done
a5c34f4 is described below
commit a5c34f4a409670c3094e3dc1ad310c3aa32bce75
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Sat Oct 27 14:46:03 2018 +0300
IGNITE-9848: Standalone syncer class; Find missing builds by refs done
---
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 12 ++------
.../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 13 ++------
.../ignited/fatbuild/ProactiveFatBuildSync.java | 35 +++++++++++++++-------
3 files changed, 29 insertions(+), 31 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 5203ac1..5603b44 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -17,8 +17,6 @@
package org.apache.ignite.ci.teamcity.ignited;
-import com.google.common.base.Strings;
-import com.google.common.base.Throwables;
import com.google.common.collect.Sets;
import org.apache.ignite.ci.ITeamcity;
import org.apache.ignite.ci.di.AutoProfiling;
@@ -28,19 +26,16 @@ import org.apache.ignite.ci.tcbot.condition.BuildCondition;
import org.apache.ignite.ci.tcbot.condition.BuildConditionDao;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync;
import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
-import org.apache.ignite.ci.util.ExceptionUtil;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import javax.inject.Inject;
-import java.io.FileNotFoundException;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@@ -84,8 +79,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
buildRefDao.init(); //todo init somehow in auto
buildConditionDao.init();
fatBuildDao.init();
-
- buildSync.invokeLaterFindMissingByBuildRef(srvNme);
}
@@ -159,6 +152,9 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
*
*/
void actualizeRecentBuildRefs() {
+ // schedule find missing later
+ buildSync.invokeLaterFindMissingByBuildRef(srvNme, conn);
+
List<BuildRefCompacted> running = buildRefDao.getQueuedAndRunning(srvIdMaskHigh);
Set<Integer> paginateUntil = new HashSet<>();
@@ -200,8 +196,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
*/
void fullReindex() {
runActualizeBuildRefs(srvNme, true, null);
-
- buildSync.invokeLaterFindMissingByBuildRef(srvNme);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index 725c074..afe4812 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -134,16 +134,7 @@ public class FatBuildDao {
return key!=null && key >> 32 == srvId;
}
- @AutoProfiling
- public int[] getAllIds(int srvId) {
- GridIntList res = new GridIntList(buildsCache.size());
-
- StreamSupport.stream(buildsCache.spliterator(), false)
- .map(Cache.Entry::getKey)
- .filter(entry -> isKeyForServer(entry, srvId))
- .map(BuildRefDao::cacheKeyToBuildId)
- .forEach(res::add);
-
- return res.array();
+ public boolean containsKey(int srvIdMaskHigh, int buildRefKey) {
+ return buildsCache.containsKey(buildIdToCacheKey(srvIdMaskHigh, buildRefKey));
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index 9d83187..8eac870 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -95,21 +95,34 @@ public class ProactiveFatBuildSync {
@SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
@MonitoredTask(name = "Find missing builds", nameExtArgsIndexes = {0})
@AutoProfiling
- protected String findMissingBuildsFromBuildRef(String srvId) {
- int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
+ protected String findMissingBuildsFromBuildRef(String srvId, ITeamcityConn conn) {
+ int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
final int[] buildRefKeys = buildRefDao.getAllIds(srvIdMaskHigh);
- final int[] fatBuildKeys = fatBuildDao.getAllIds(srvIdMaskHigh);
Arrays.parallelSort(buildRefKeys);
- Arrays.parallelSort(fatBuildKeys);
- /* ;
- */
- return "";
- }
+ List<Integer> buildsIdsToLoad = new ArrayList<>();
+ int totalAskedToLoad = 0;
+
+ for (int buildRefKey : buildRefKeys) {
+ if (!fatBuildDao.containsKey(srvIdMaskHigh, buildRefKey))
+ buildsIdsToLoad.add(buildRefKey);
+ if (buildsIdsToLoad.size() >= 100) {
+ totalAskedToLoad += buildsIdsToLoad.size();
+ scheduleBuildsLoad(conn, buildsIdsToLoad);
+ buildsIdsToLoad.clear();
+ }
+ }
+ if (!buildsIdsToLoad.isEmpty()) {
+ totalAskedToLoad += buildsIdsToLoad.size();
+ scheduleBuildsLoad(conn, buildsIdsToLoad);
+ }
+
+ return "Invoked later load for " + totalAskedToLoad + " builds from " + srvId;
+ }
/** */
private void loadFatBuilds(int ldrNo, String serverId) {
@@ -139,7 +152,7 @@ public class ProactiveFatBuildSync {
}
@SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
- @MonitoredTask(name = "Proactive Builds Loading (agent,server)", nameExtArgsIndexes = {0, 1})
+ @MonitoredTask(name = "Proactive Builds Loading (srv,agent)", nameExtArgsIndexes = {1, 0})
@AutoProfiling
protected String doLoadBuilds(int ldrNo, String srvId, ITeamcityConn conn, Set<Integer> load) {
if(load.isEmpty())
@@ -177,9 +190,9 @@ public class ProactiveFatBuildSync {
return ProactiveFatBuildSync.class.getSimpleName() +"." + taskName + "." + srvName;
}
- public void invokeLaterFindMissingByBuildRef(String srvName) {
+ public void invokeLaterFindMissingByBuildRef(String srvName, ITeamcityConn conn) {
scheduler.sheduleNamed(taskName("findMissingBuildsFromBuildRef", srvName),
- () -> findMissingBuildsFromBuildRef(srvName), 360, TimeUnit.MINUTES);
+ () -> findMissingBuildsFromBuildRef(srvName, conn), 360, TimeUnit.MINUTES);
}
/**