You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/11/30 16:06:03 UTC
[ignite-teamcity-bot] branch ignite-10071 updated: IGNITE-10071
First case of queued build in refs but not in fat builds fixed
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-10071
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-10071 by this push:
new 4673b01 IGNITE-10071 First case of queued build in refs but not in fat builds fixed
4673b01 is described below
commit 4673b0121631d6a08ea2869b8157d9e7e93085d9
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Nov 30 19:06:01 2018 +0300
IGNITE-10071 First case of queued build in refs but not in fat builds fixed
---
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 2 +
.../ignite/ci/teamcity/ignited/BuildRefDao.java | 14 +++---
.../ci/teamcity/ignited/ITeamcityIgnited.java | 5 +-
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 12 +----
.../ignited/fatbuild/FatBuildCompacted.java | 4 +-
.../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 8 ++--
.../ignited/fatbuild/ProactiveFatBuildSync.java | 53 +++++++++++++++++-----
.../ignite/ci/teamcity/pure/ITeamcityConn.java | 2 +-
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 40 +++++++++++-----
9 files changed, 94 insertions(+), 46 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index 8ebc749..4b256ce 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -89,6 +89,8 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
* Teamcity
*/
private ITeamcity teamcity;
+
+ @Nullable
private String serverId;
/** cached running builds for branch. */
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
index cbb4645..de96fdc 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
@@ -55,19 +55,21 @@ public class BuildRefDao {
@Inject private IStringCompactor compactor;
/** */
- public void init() {
+ public BuildRefDao init() {
CacheConfiguration<Long, BuildRefCompacted> cfg = TcHelperDb.getCacheV2Config(TEAMCITY_BUILD_CACHE_NAME);
cfg.setQueryEntities(Collections.singletonList(new QueryEntity(Long.class, BuildRefCompacted.class)));
buildRefsCache = igniteProvider.get().getOrCreateCache(cfg);
+
+ return this;
}
/**
* @param srvId Server id.
* @return all builds for a server, full scan.
*/
- @NotNull protected Stream<BuildRefCompacted> compactedBuildsForServer(int srvId) {
+ @NotNull public Stream<BuildRefCompacted> compactedBuildsForServer(int srvId) {
return StreamSupport.stream(buildRefsCache.spliterator(), false)
.filter(entry -> isKeyForServer(entry.getKey(), srvId))
.map(javax.cache.Cache.Entry::getValue);
@@ -228,16 +230,16 @@ public class BuildRefDao {
public int[] getAllIds(int srvId) {
GridIntList res = new GridIntList(buildRefsCache.size());
- getAllBuilds(srvId)
+ getAllBuildRefs(srvId)
+ .map(Cache.Entry::getKey)
.map(BuildRefDao::cacheKeyToBuildId)
.forEach(res::add);
return res.array();
}
- @NotNull public Stream<Long> getAllBuilds(int srvId) {
+ @NotNull public Stream<Cache.Entry<Long, BuildRefCompacted>> getAllBuildRefs(int srvId) {
return StreamSupport.stream(buildRefsCache.spliterator(), false)
- .map(Cache.Entry::getKey)
- .filter(entry -> isKeyForServer(entry, srvId));
+ .filter(entry -> isKeyForServer(entry.getKey(), srvId));
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
index bc3788c..bfd5bc9 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
@@ -84,7 +84,10 @@ public interface ITeamcityIgnited {
* @param srvId Server id.
* @return integer representation of server ID.
*/
- public static int serverIdToInt(String srvId) {
+ public static int serverIdToInt(@Nullable final String srvId) {
+ if (srvId == null)
+ return 0;
+
return Math.abs(srvId.hashCode());
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index df11962..1d0b4bc 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -443,13 +443,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
if (mode == SyncMode.NONE) {
if (existingBuild != null)
return existingBuild;
- else {
- FatBuildCompacted buildCompacted = new FatBuildCompacted();
-
- buildCompacted.setFakeStub(true);
-
- return buildCompacted; // providing fake builds
- }
+ else
+ return new FatBuildCompacted().setFakeStub(true); // providing fake builds
}
FatBuildCompacted savedVer = buildSync.loadBuild(conn, buildId, existingBuild, mode);
@@ -458,9 +453,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
if (savedVer == null)
return existingBuild;
- buildRefDao.save(srvIdMaskHigh, new BuildRefCompacted(savedVer));
- runHistSync.saveToHistoryLater(srvNme, savedVer);
-
return savedVer;
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index fd153e9..1a50a5d 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -158,8 +158,10 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
setFakeStub(true);
}
- public void setFakeStub(boolean val) {
+ public FatBuildCompacted setFakeStub(boolean val) {
setFlag(FAKE_BUILD_F, val);
+
+ return this;
}
public void buildTypeName(String btName, IStringCompactor compactor) {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index 2c4df80..0e2e4bd 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -63,8 +63,10 @@ public class FatBuildDao {
/**
*
*/
- public void init() {
+ public FatBuildDao init() {
buildsCache = igniteProvider.get().getOrCreateCache(TcHelperDb.getCacheV2Config(TEAMCITY_FAT_BUILD_CACHE_NAME));
+
+ return this;
}
/**
@@ -172,7 +174,7 @@ public class FatBuildDao {
return key!=null && key >> 32 == srvId;
}
- public boolean containsKey(int srvIdMaskHigh, int buildRefKey) {
- return buildsCache.containsKey(buildIdToCacheKey(srvIdMaskHigh, buildRefKey));
+ public boolean containsKey(int srvIdMaskHigh, int buildId) {
+ return buildsCache.containsKey(buildIdToCacheKey(srvIdMaskHigh, buildId));
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index 3cd1a5d..03e6311 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -18,6 +18,7 @@ package org.apache.ignite.ci.teamcity.ignited.fatbuild;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
+import java.util.stream.Stream;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.di.MonitoredTask;
import org.apache.ignite.ci.di.scheduler.IScheduler;
@@ -26,11 +27,13 @@ import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.teamcity.ignited.SyncMode;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistSync;
import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
import org.apache.ignite.ci.util.ExceptionUtil;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
@@ -65,8 +68,12 @@ public class ProactiveFatBuildSync {
@Inject private IStringCompactor compactor;
+ /** Change sync. */
@Inject private ChangeSync changeSync;
+ /** Run history sync. */
+ @Inject private RunHistSync runHistSync;
+
@GuardedBy("this")
private Map<String, SyncTask> buildToLoad = new HashMap<>();
@@ -124,30 +131,34 @@ public class ProactiveFatBuildSync {
protected String findMissingBuildsFromBuildRef(String srvId, ITeamcityConn conn) {
int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
- final int[] buildRefKeys = buildRefDao.getAllIds(srvIdMaskHigh);
+ Stream<BuildRefCompacted> buildRefs = buildRefDao.compactedBuildsForServer(srvIdMaskHigh);
List<Integer> buildsIdsToLoad = new ArrayList<>();
- int totalAskedToLoad = 0;
+ AtomicInteger totalAskedToLoad = new AtomicInteger();
+
+ buildRefs.forEach(buildRef -> {
+ Integer buildId = buildRef.getId();
+ if (buildId == null)
+ return;
- for (int buildRefKey : buildRefKeys) {
//todo
// !existingBuild.isRunning(compactor) && !existingBuild.isQueued(compactor);
- if (!fatBuildDao.containsKey(srvIdMaskHigh, buildRefKey))
- buildsIdsToLoad.add(buildRefKey);
+ if (!fatBuildDao.containsKey(srvIdMaskHigh, buildId))
+ buildsIdsToLoad.add(buildId);
if (buildsIdsToLoad.size() >= 100) {
- totalAskedToLoad += buildsIdsToLoad.size();
+ totalAskedToLoad.addAndGet(buildsIdsToLoad.size());
scheduleBuildsLoad(conn, buildsIdsToLoad);
buildsIdsToLoad.clear();
}
- }
+ });
if (!buildsIdsToLoad.isEmpty()) {
- totalAskedToLoad += buildsIdsToLoad.size();
+ totalAskedToLoad.addAndGet(buildsIdsToLoad.size());
scheduleBuildsLoad(conn, buildsIdsToLoad);
}
- return "Invoked later load for " + totalAskedToLoad + " builds from " + srvId;
+ return "Invoked later load for " + totalAskedToLoad.get() + " builds from " + srvId;
}
/** */
@@ -250,7 +261,20 @@ public class ProactiveFatBuildSync {
return null;
}
- return reloadBuild(conn, buildId, existingBuild);
+ FatBuildCompacted savedVer = reloadBuild(conn, buildId, existingBuild);
+
+ BuildRefCompacted refCompacted = new BuildRefCompacted(savedVer);
+ if (savedVer.isFakeStub())
+ refCompacted.setId(buildId); //to provide possiblity to save the build
+
+ final String srvNme = conn.serverId();
+ final int srvIdMask = ITeamcityIgnited.serverIdToInt(srvNme);
+
+ buildRefDao.save(srvIdMask, refCompacted);
+
+ runHistSync.saveToHistoryLater(srvNme, savedVer);
+
+ return savedVer;
}
/**
@@ -277,6 +301,9 @@ public class ProactiveFatBuildSync {
try {
build = conn.getBuild(buildId);
+ if (build.isFakeStub())
+ build.setCancelled();
+
if(build.testOccurrences != null && !build.isComposite()) { // don't query tests for compoite
String nextHref = null;
do {
@@ -318,9 +345,11 @@ public class ProactiveFatBuildSync {
problems = existingBuild.problems(compactor);
}
- else
+ else {
build = Build.createFakeStub();
- //todo here can be situation we have build ref, but don't have a build
+
+ build.setCancelled();
+ }
} else {
logger.error("Loading build [" + buildId + "] for server [" + srvNme + "] failed:" + e.getMessage(), e);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
index c2884ed..5538bce 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
@@ -39,7 +39,7 @@ public interface ITeamcityConn {
/**
* @return Internal server ID as string
*/
- String serverId();
+ @Nullable public String serverId();
/**
* @return Normalized Host address, ends with '/'.
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index a9140e5..eeebd5e 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -16,6 +16,8 @@
*/
package org.apache.ignite.ci.teamcity.ignited;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
@@ -71,6 +73,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
+import static com.google.common.base.Preconditions.checkNotNull;
import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertFalse;
import static junit.framework.TestCase.assertNotNull;
@@ -549,9 +552,8 @@ public class IgnitedTcInMemoryIntegrationTest {
});
Injector injector = Guice.createInjector(module, new IgniteAndShedulerTestModule());
IStringCompactor c = injector.getInstance(IStringCompactor.class);
- BuildRefDao buildRefDao = injector.getInstance(BuildRefDao.class);
- buildRefDao.init();
- injector.getInstance(FatBuildDao.class).init();
+ BuildRefDao buildRefDao = injector.getInstance(BuildRefDao.class).init();
+ FatBuildDao fatBuildDao = injector.getInstance(FatBuildDao.class).init();
int buildId = 1000042;
BuildRef ref = new BuildRef();
@@ -559,25 +561,39 @@ public class IgnitedTcInMemoryIntegrationTest {
ref.branchName = ITeamcity.REFS_HEADS_MASTER;
ref.state = BuildRef.STATE_QUEUED;
ref.setId(buildId);
+
+ int buildIdRunning = 1000043;
+ BuildRef refR = new BuildRef();
+ refR.buildTypeId = "Testbuild";
+ refR.branchName = ITeamcity.REFS_HEADS_MASTER;
+ refR.state = BuildRef.STATE_RUNNING;
+ refR.setId(buildIdRunning);
+
String srvId = APACHE;
int srvIdInt = ITeamcityIgnited.serverIdToInt(srvId);
- buildRefDao.saveChunk(srvIdInt, Collections.singletonList(ref));
-
- ITcServerFactory srvFactory = injector.getInstance(ITcServerFactory.class);
- IAnalyticsEnabledTeamcity srv = srvFactory.createServer(srvId);
+ buildRefDao.saveChunk(srvIdInt, Lists.newArrayList(ref, refR));
List<BuildRefCompacted> running = buildRefDao.getQueuedAndRunning(srvIdInt);
- assertNotNull(running);
- assertFalse(running.isEmpty());
+ assertFalse(checkNotNull(running).isEmpty());
System.out.println("Running builds: " + running);
ProactiveFatBuildSync buildSync = injector.getInstance(ProactiveFatBuildSync.class);
- buildSync.invokeLaterFindMissingByBuildRef(srvId, srv);
+ buildSync.invokeLaterFindMissingByBuildRef(srvId,
+ injector.getInstance(ITcServerFactory.class).createServer(srvId));
+
+ FatBuildCompacted fatBuild = fatBuildDao.getFatBuild(srvIdInt, buildId);
+ System.out.println(fatBuild);
+
+ assertNotNull(fatBuild);
+ assertTrue(fatBuild.isFakeStub());
+
+ assertTrue(fatBuild.isCancelled(c));
List<BuildRefCompacted> running2 = buildRefDao.getQueuedAndRunning(srvIdInt);
- assertNotNull(running2);
- assertTrue(running2.isEmpty());
+ System.out.println("Running builds (after sync): " +
+ running2.stream().map(bref->bref.toBuildRef(c)).collect(Collectors.toList()));
+ assertTrue(checkNotNull(running2).isEmpty());
}
/**