You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/06/21 19:01:25 UTC
[ignite-teamcity-bot] branch test-hist-performance updated (c802af7
-> 6e19099)
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a change to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git.
from c802af7 Trusted tests & suite history performance fixes: tests fix, disable of call to old hist services
new 1278c67 Trusted tests & suite history performance fixes: tests fix, sharing suite based history
new 6e19099 Trusted tests & suite history performance fixes: fix bug with race
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../ignite/ci/tcbot/issue/IssueDetector.java | 2 +-
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
.../ci/tcbot/chain/PrChainsProcessorTest.java | 1 +
.../ci/teamcity/ignited/TeamcityIgnitedMock.java | 36 +++++++++++++++++++
.../tcbot/engine/chain/BuildChainProcessor.java | 42 ++++++++++++++--------
.../ignite/tcbot/engine/chain/MultBuildRunCtx.java | 38 ++++++++++++++++----
.../tcbot/engine/chain/TestCompactedMult.java | 21 +++++++----
.../ignite/tcbot/engine/pr/PrChainsProcessor.java | 5 ++-
.../apache/ignite/tcbot/engine/ui/DsSuiteUi.java | 9 ++---
.../ignite/tcbot/engine/ui/DsTestFailureUi.java | 8 ++---
.../apache/ignite/tcignited/ITeamcityIgnited.java | 3 +-
.../ignite/tcignited/TeamcityIgnitedImpl.java | 3 +-
.../ignite/tcignited/build/SuiteHistory.java | 11 ++++--
.../ignite/tcignited/history/HistoryCollector.java | 29 ++++++++++-----
...{IRunHistSummary.java => ISuiteRunHistory.java} | 16 ++-------
.../tcignited/history/RunHistCompactedDao.java | 4 +--
.../history/SuiteInvocationHistoryDao.java | 6 ++--
17 files changed, 161 insertions(+), 75 deletions(-)
copy tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/{IRunHistSummary.java => ISuiteRunHistory.java} (84%)
[ignite-teamcity-bot] 02/02: Trusted tests & suite history
performance fixes: fix bug with race
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 6e190994dfadf6363b4e12d02ba64533b59ffe29
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Jun 21 22:01:12 2019 +0300
Trusted tests & suite history performance fixes: fix bug with race
---
.../org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java | 2 --
.../org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java | 4 ++--
.../org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java | 8 +++++++-
3 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java
index fcfe597..fc1de27 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java
@@ -108,8 +108,6 @@ public class PrChainsProcessorTest {
//todo flaky test
@Test
- @Ignore
- //todo fix failing test
public void testTestFailureWithoutStatReportedAsBlocker() {
IStringCompactor c = injector.getInstance(IStringCompactor.class);
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
index d4d6ed9..579dbb0 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
@@ -185,13 +185,13 @@ public class BuildChainProcessor {
final MultBuildRunCtx ctx = new MultBuildRunCtx(ref, compactor);
+ buildsForSuite.forEach(buildCompacted -> ctx.addBuild(loadChanges(buildCompacted, tcIgn)));
+
//ask for history for the suite in parallel
tcUpdatePool.getService().submit(() -> {
ctx.history(tcIgn, failRateBranchId);
});
- buildsForSuite.forEach(buildCompacted -> ctx.addBuild(loadChanges(buildCompacted, tcIgn)));
-
analyzeTests(ctx, tcIgn, procLog);
fillBuildCounts(ctx, tcIgn, includeScheduledInfo);
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
index ad86342..f965db3 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
@@ -17,6 +17,7 @@
package org.apache.ignite.tcbot.engine.chain;
+import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.cache.CacheBuilder;
import java.util.ArrayList;
@@ -659,9 +660,14 @@ public class MultBuildRunCtx implements ISuiteResults {
@Nullable
ISuiteRunHistory suiteHist(ITeamcityIgnited tcIgn, Integer baseBranchId) {
+ Integer buildTypeIdId = buildTypeIdId();
+ Preconditions.checkNotNull(buildTypeIdId, "Build type ID should be filled");
+
try {
return historyCacheMap.get(baseBranchId,
- () -> Optional.ofNullable(tcIgn.getSuiteRunHist(buildTypeIdId(), baseBranchId)))
+ () -> {
+ return Optional.ofNullable(tcIgn.getSuiteRunHist(buildTypeIdId, baseBranchId));
+ })
.orElse(null);
}
catch (ExecutionException e) {
[ignite-teamcity-bot] 01/02: Trusted tests & suite history
performance fixes: tests fix, sharing suite based history
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 1278c67a0c23002859b71a094ae8850960f4dfcb
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Jun 21 21:52:49 2019 +0300
Trusted tests & suite history performance fixes: tests fix, sharing suite based history
---
.../ignite/ci/tcbot/issue/IssueDetector.java | 2 +-
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
.../ci/tcbot/chain/PrChainsProcessorTest.java | 3 ++
.../ci/teamcity/ignited/TeamcityIgnitedMock.java | 36 +++++++++++++++++++
.../tcbot/engine/chain/BuildChainProcessor.java | 42 ++++++++++++++--------
.../ignite/tcbot/engine/chain/MultBuildRunCtx.java | 32 +++++++++++++----
.../tcbot/engine/chain/TestCompactedMult.java | 21 +++++++----
.../ignite/tcbot/engine/pr/PrChainsProcessor.java | 5 ++-
.../apache/ignite/tcbot/engine/ui/DsSuiteUi.java | 9 ++---
.../ignite/tcbot/engine/ui/DsTestFailureUi.java | 8 ++---
.../apache/ignite/tcignited/ITeamcityIgnited.java | 3 +-
.../ignite/tcignited/TeamcityIgnitedImpl.java | 3 +-
.../ignite/tcignited/build/SuiteHistory.java | 11 ++++--
.../ignite/tcignited/history/HistoryCollector.java | 29 ++++++++++-----
.../ignite/tcignited/history/ISuiteRunHistory.java | 22 ++++++++++++
.../tcignited/history/RunHistCompactedDao.java | 4 +--
.../history/SuiteInvocationHistoryDao.java | 6 ++--
17 files changed, 176 insertions(+), 62 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
index ead0965..62842f0 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
@@ -295,7 +295,7 @@ public class IssueDetector {
Integer btId = compactor.getStringIdIfPresent(suiteId);
Integer brNormId = compactor.getStringIdIfPresent(normalizeBranch);
- IRunHistory runStat = tcIgnited.getSuiteRunHist(btId, brNormId);
+ IRunHistory runStat = tcIgnited.getSuiteRunHist(btId, brNormId).self();
if (runStat == null)
return false;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
index dcc646e..c1bd22d 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
@@ -28,7 +28,7 @@ package org.apache.ignite.ci.web.model;
public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot";
/** TC Bot Version. */
- public static final String VERSION = "20190613";
+ public static final String VERSION = "20190621";
/** Java version, where Web App is running. */
public String javaVer;
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java
index f38ce55..fcfe597 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java
@@ -47,6 +47,7 @@ import org.apache.ignite.tcbot.engine.ui.DsSuiteUi;
import org.apache.ignite.tcbot.engine.ui.DsTestFailureUi;
import org.jetbrains.annotations.NotNull;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import static org.junit.Assert.assertFalse;
@@ -107,6 +108,8 @@ public class PrChainsProcessorTest {
//todo flaky test
@Test
+ @Ignore
+ //todo fix failing test
public void testTestFailureWithoutStatReportedAsBlocker() {
IStringCompactor c = injector.getInstance(IStringCompactor.class);
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
index c744616..9a0d174 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
@@ -31,6 +31,7 @@ import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.SyncMode;
+import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
import org.jetbrains.annotations.NotNull;
import org.mockito.Mockito;
@@ -118,6 +119,41 @@ public class TeamcityIgnitedMock {
return runHistCompacted;
});
+ when(tcIgnited.getSuiteRunHist(anyInt(), anyInt()))
+ .thenAnswer((inv) -> {
+ final Integer suiteName = inv.getArgument(0);
+ final Integer branchId = inv.getArgument(1);
+ // System.out.println("Search history " + name + " in " + branch + ": " );
+ if (histCache.isEmpty()) {
+ synchronized (histCache) {
+ if (histCache.isEmpty())
+ initHistory(c, histCache, builds, srvId);
+ }
+ }
+
+ ISuiteRunHistory mock = Mockito.mock(ISuiteRunHistory.class);
+
+ when(mock.getTestRunHist(anyInt())).thenAnswer((inv2)-> {
+ final Integer tstName = inv2.getArgument(0);
+
+ if (tstName == null)
+ return null;
+
+ if (branchId == null)
+ return null;
+
+ final RunHistKey key = new RunHistKey(srvId, tstName, branchId);
+
+ final RunHistCompacted runHistCompacted = histCache.get(key);
+
+ System.out.println("Test history " + c.getStringFromId(tstName) + " in " + c.getStringFromId(branchId) + " => " + runHistCompacted);
+
+ return runHistCompacted;
+ });
+
+ return mock;
+ });
+
// when(tcIgnited.gitBranchPrefix()).thenReturn("ignite-");
ITcServerConfig mock = mock(ITcServerConfig.class);
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
index 144c6ef..d4d6ed9 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
@@ -19,12 +19,32 @@ package org.apache.ignite.tcbot.engine.chain;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.Futures;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.inject.Inject;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.buildtype.ParametersCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.util.FutureUtil;
import org.apache.ignite.tcbot.engine.pool.TcUpdatePool;
-import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.engine.ui.LrTestUi;
import org.apache.ignite.tcbot.engine.ui.LrTestsSuiteSummaryUi;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
@@ -38,18 +58,6 @@ import org.apache.ignite.tcservice.model.result.Build;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-import java.util.*;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Future;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-
/**
* Process whole Build Chain, E.g. runAll at particular server, including all builds involved
*/
@@ -177,6 +185,11 @@ public class BuildChainProcessor {
final MultBuildRunCtx ctx = new MultBuildRunCtx(ref, compactor);
+ //ask for history for the suite in parallel
+ tcUpdatePool.getService().submit(() -> {
+ ctx.history(tcIgn, failRateBranchId);
+ });
+
buildsForSuite.forEach(buildCompacted -> ctx.addBuild(loadChanges(buildCompacted, tcIgn)));
analyzeTests(ctx, tcIgn, procLog);
@@ -230,7 +243,8 @@ public class BuildChainProcessor {
.peek(val -> Preconditions.checkNotNull(val, "Build future should be in context"))
.flatMap(ref -> dependencies(ref, mode, builds, tcIgn).stream()).collect(Collectors.toSet());
- logger.info("Level [" + level + "] dependencies:" + depsNextLevel);
+ if(logger.isDebugEnabled())
+ logger.debug("Level [" + level + "] dependencies:" + depsNextLevel);
remainedUnloadedDeps = depsNextLevel;
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
index 232b88d..ad86342 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
@@ -18,6 +18,7 @@
package org.apache.ignite.tcbot.engine.chain;
import com.google.common.base.Strings;
+import com.google.common.cache.CacheBuilder;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
@@ -27,8 +28,8 @@ import java.util.Objects;
import java.util.Optional;
import java.util.OptionalDouble;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@@ -39,12 +40,14 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
+import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
import org.apache.ignite.tcbot.common.util.CollectionUtil;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.buildlog.ILogCheckResult;
import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
import org.apache.ignite.tcignited.history.IRunHistory;
+import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcservice.model.hist.BuildRef;
import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence;
import org.apache.ignite.tcservice.model.result.stat.Statistics;
@@ -67,7 +70,8 @@ public class MultBuildRunCtx implements ISuiteResults {
/** Builds: Single execution. */
private List<SingleBuildRunCtx> builds = new CopyOnWriteArrayList<>();
- private java.util.Map<Integer, IRunHistory> historyCacheMap = new ConcurrentHashMap<>();
+ private final com.google.common.cache.Cache<Integer, Optional<ISuiteRunHistory>> historyCacheMap
+ = CacheBuilder.newBuilder().build();
public void addBuild(SingleBuildRunCtx ctx) {
builds.add(ctx);
@@ -331,7 +335,7 @@ public class MultBuildRunCtx implements ISuiteResults {
public void saveToMap(Map<Integer, TestCompactedMult> res, Stream<TestCompacted> tests) {
tests.forEach(testCompacted -> {
- res.computeIfAbsent(testCompacted.testName(), k -> new TestCompactedMult(compactor))
+ res.computeIfAbsent(testCompacted.testName(), k -> new TestCompactedMult(compactor, this))
.add(testCompacted);
});
}
@@ -615,10 +619,9 @@ public class MultBuildRunCtx implements ISuiteResults {
singleBuildRunCtx.getAllTests().filter(t -> !t.isIgnoredTest() && !t.isMutedTest()));
});
Integer branchName = compactor.getStringIdIfPresent(normalizedBaseBranch);
- Integer suiteName = buildTypeIdId();
res.forEach((testNameId, compactedMult) -> {
- IRunHistory stat = compactedMult.history(tcIgnited, suiteName, branchName);
+ IRunHistory stat = compactedMult.history(tcIgnited, branchName);
String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
boolean b = testBlockerComment != null;
if (b) // this test will be considered as blocker if will fail
@@ -647,7 +650,22 @@ public class MultBuildRunCtx implements ISuiteResults {
if (baseBranchId == null)
return null;
- return historyCacheMap.computeIfAbsent(baseBranchId,
- (k) -> tcIgn.getSuiteRunHist(buildTypeIdId(), k));
+ ISuiteRunHistory suiteHist = suiteHist(tcIgn, baseBranchId);
+ if (suiteHist == null)
+ return null;
+
+ return suiteHist.self();
+ }
+
+ @Nullable
+ ISuiteRunHistory suiteHist(ITeamcityIgnited tcIgn, Integer baseBranchId) {
+ try {
+ return historyCacheMap.get(baseBranchId,
+ () -> Optional.ofNullable(tcIgn.getSuiteRunHist(buildTypeIdId(), baseBranchId)))
+ .orElse(null);
+ }
+ catch (ExecutionException e) {
+ throw ExceptionUtil.propagateException(e);
+ }
}
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
index b1ab48f..7d5d3c3 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java
@@ -20,7 +20,6 @@ package org.apache.ignite.tcbot.engine.chain;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
@@ -29,6 +28,7 @@ import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.history.IRunHistSummary;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.IRunStat;
+import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
/**
@@ -37,11 +37,12 @@ import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
public class TestCompactedMult implements IMultTestOccurrence {
private final List<TestCompacted> occurrences = new ArrayList<>();
private IStringCompactor compactor;
+ private MultBuildRunCtx ctx;
private long avgDuration = -1;
- private java.util.Map<Integer, IRunHistory> historyCacheMap = new ConcurrentHashMap<>();
- public TestCompactedMult(IStringCompactor compactor) {
+ public TestCompactedMult(IStringCompactor compactor, MultBuildRunCtx ctx) {
this.compactor = compactor;
+ this.ctx = ctx;
}
@Nullable public Integer testName() {
@@ -119,8 +120,16 @@ public class TestCompactedMult implements IMultTestOccurrence {
}
- public IRunHistory history(ITeamcityIgnited ignited, Integer buildTypeIdId, Integer baseBranchId) {
- return historyCacheMap.computeIfAbsent(baseBranchId,
- (k)-> ignited.getTestRunHist(testName(), buildTypeIdId, k));
+ public IRunHistory history(ITeamcityIgnited ignited, Integer baseBranchId) {
+ Integer name = testName();
+ if (name == null)
+ return null;
+
+ ISuiteRunHistory suiteRunHist = ctx.suiteHist(ignited, baseBranchId);
+
+ if (suiteRunHist == null)
+ return null;
+
+ return suiteRunHist.getTestRunHist(name);
}
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
index 2da9f5e..5a9246c 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
@@ -260,18 +260,17 @@ public class PrChainsProcessor {
String normalizedBaseBranch = RunHistSync.normalizeBranch(baseBranch);
Integer baseBranchId = compactor.getStringIdIfPresent(normalizedBaseBranch);
IRunHistory statInBaseBranch = ctx.history(tcIgnited, baseBranchId);
- Integer suiteId = compactor.getStringIdIfPresent(ctx.suiteId()); // can be inlined
String suiteComment = ctx.getPossibleBlockerComment(compactor, statInBaseBranch, tcIgnited.config());
List<DsTestFailureUi> failures = ctx.getFailedTests().stream().map(occurrence -> {
- IRunHistory stat = occurrence.history(tcIgnited, suiteId, baseBranchId);
+ IRunHistory stat = occurrence.history(tcIgnited, baseBranchId);
String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
if (!Strings.isNullOrEmpty(testBlockerComment)) {
final DsTestFailureUi failure = new DsTestFailureUi();
- failure.initFromOccurrence(occurrence, ctx.buildTypeIdId(), tcIgnited, ctx.projectId(), ctx.branchName(), baseBranch, baseBranchId);
+ failure.initFromOccurrence(occurrence, tcIgnited, ctx.projectId(), ctx.branchName(), baseBranch, baseBranchId);
return failure;
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
index c823fd9..eea9212 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
@@ -192,7 +192,7 @@ public class DsSuiteUi extends DsHistoryStatUi {
if (includeTests) {
List<TestCompactedMult> tests = suite.getFailedTests();
Function<TestCompactedMult, Float> function = testCompactedMult -> {
- IRunHistory res = testCompactedMult.history(tcIgnited, buildTypeIdId, baseBranchId);
+ IRunHistory res = testCompactedMult.history(tcIgnited, baseBranchId);
return res == null ? 0f : res.getFailRate();
};
@@ -201,7 +201,7 @@ public class DsSuiteUi extends DsHistoryStatUi {
tests.forEach(occurrence -> {
final DsTestFailureUi failure = new DsTestFailureUi();
- failure.initFromOccurrence(occurrence, buildTypeIdId, tcIgnited, suite.projectId(),
+ failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(),
suite.branchName(), baseBranch, baseBranchId);
failure.initStat(occurrence, buildTypeIdId, tcIgnited, baseBranchId, curBranchId);
@@ -267,9 +267,6 @@ public class DsSuiteUi extends DsHistoryStatUi {
Integer failRateNormalizedBranch,
Integer curBranchNormalized,
MultBuildRunCtx suite) {
- if (suite.buildTypeIdId() == null)
- return null;
-
IRunHistory statInBaseBranch = suite.history(tcIgnited, failRateNormalizedBranch);
if (statInBaseBranch != null) {
@@ -325,7 +322,7 @@ public class DsSuiteUi extends DsHistoryStatUi {
Integer baseBranchId = compactor.getStringIdIfPresent(normalizeBranch(failRateBranch));
Integer buildTypeIdId = suite.buildTypeIdId();
- failure.initFromOccurrence(occurrence, buildTypeIdId, tcIgnited, suite.projectId(), suite.branchName(),
+ failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(),
failRateBranch, baseBranchId);
failure.initStat(occurrence, buildTypeIdId, tcIgnited, baseBranchId,
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java
index 9c146f9..6f9f127 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsTestFailureUi.java
@@ -90,7 +90,6 @@ public class DsTestFailureUi {
/**
* @param failure test ocurrence (probably multiple)
- * @param buildTypeIdId
* @param tcIgn Teamcity.
* @param projectId project ID.
* @param branchName current branch name.
@@ -98,7 +97,6 @@ public class DsTestFailureUi {
* @param baseBranchId Normalized base branch ID (from compactor).
*/
public void initFromOccurrence(@Nonnull final TestCompactedMult failure,
- Integer buildTypeIdId,
@Nonnull final ITeamcityIgnited tcIgn,
@Nullable final String projectId,
@Nullable final String branchName,
@@ -146,7 +144,7 @@ public class DsTestFailureUi {
webUrlBaseBranch = buildWebLink(tcIgn, full.test.id, projectId, baseBranchName);
});
- final IRunHistory stat = failure.history(tcIgn, buildTypeIdId, baseBranchId);
+ final IRunHistory stat = failure.history(tcIgn, baseBranchId);
blockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
}
@@ -201,13 +199,13 @@ public class DsTestFailureUi {
public void initStat(TestCompactedMult occurrence, Integer buildTypeIdId, ITeamcityIgnited tcIgnited,
@Nullable Integer baseBranchId,
@Nullable Integer curBranchNormalized) {
- final IRunHistory stat = occurrence.history(tcIgnited, buildTypeIdId, baseBranchId);
+ final IRunHistory stat = occurrence.history(tcIgnited, baseBranchId);
histBaseBranch.init(stat);
IRunHistory statForProblemsDetection;
if (!Objects.equals(curBranchNormalized, baseBranchId)) {
- statForProblemsDetection = occurrence.history(tcIgnited, buildTypeIdId, curBranchNormalized);
+ statForProblemsDetection = occurrence.history(tcIgnited, curBranchNormalized);
if (statForProblemsDetection != null) {
histCurBranch = new DsTestHistoryUi();
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
index 7cd950e..5ffd315 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
@@ -33,6 +33,7 @@ import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.IRunStat;
+import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcservice.model.agent.Agent;
import org.apache.ignite.tcservice.model.mute.MuteInfo;
import org.apache.ignite.tcservice.model.result.Build;
@@ -206,7 +207,7 @@ public interface ITeamcityIgnited {
@Deprecated
@Nullable public IRunHistory getSuiteRunHist(String suiteId, @Nullable String branch);
- @Nullable public IRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch);
+ @Nullable public ISuiteRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch);
/**
* V.3.0 run history implementation based on scan of fat builds.
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index f15ecab..ea68f59 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -63,6 +63,7 @@ import org.apache.ignite.tcignited.buildref.BuildRefSync;
import org.apache.ignite.tcignited.history.HistoryCollector;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.IRunStat;
+import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcignited.history.RunHistCompactedDao;
import org.apache.ignite.tcignited.history.RunHistSync;
import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
@@ -436,7 +437,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
/** {@inheritDoc} */
- @Nullable @Override public IRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch) {
+ @Nullable @Override public ISuiteRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch) {
if (buildTypeId == null || normalizedBaseBranch == null)
return null;
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
index cfd24fe..0e2d010 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
@@ -25,12 +25,13 @@ import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
import org.apache.ignite.internal.binary.BinaryObjectExImpl;
import org.apache.ignite.tcignited.history.IRunHistory;
+import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcignited.history.SuiteInvocation;
/**
* Suite run history summary.
*/
-public class SuiteHistory {
+public class SuiteHistory implements ISuiteRunHistory {
/** Tests history: Test name ID->RunHistory */
private Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
@@ -41,8 +42,8 @@ public class SuiteHistory {
return binary.length();
}
- public IRunHistory getTestRunHist(int name) {
- return testsHistory.get(name);
+ public IRunHistory getTestRunHist(int testName) {
+ return testsHistory.get(testName);
}
public RunHistCompacted getOrAddTestsHistory(Integer tName) {
@@ -62,4 +63,8 @@ public class SuiteHistory {
public RunHistCompacted getSuiteHist() {
return suiteHist;
}
+
+ @Override public IRunHistory self() {
+ return suiteHist;
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
index 4a62534..7254774 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -16,6 +16,7 @@
*/
package org.apache.ignite.tcignited.history;
+import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
import java.time.Duration;
@@ -139,12 +140,13 @@ public class HistoryCollector {
List<BuildRefCompacted> bRefsList = buildRefDao.getAllBuildsCompacted(srvId, btId,
branchEquivalence.branchForQuery(branchId));
- AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId);
- int outOfScopeBuildId = biggestIdOutOfScope == null ? -1 : biggestIdOutOfScope.get();
-
long curTs = System.currentTimeMillis();
Set<Integer> buildIds = bRefsList.stream()
- .filter(b -> b.id() > outOfScopeBuildId)
+ .filter(b -> {
+ AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId);
+ int outOfScopeBuildId = biggestIdOutOfScope == null ? -1 : biggestIdOutOfScope.get();
+ return b.id() > outOfScopeBuildId;
+ })
.filter(this::applicableForHistory)
.map(BuildRefCompacted::id)
.filter(bId -> !knownBuilds.contains(bId)).collect(Collectors.toSet());
@@ -163,7 +165,7 @@ public class HistoryCollector {
buildStartTimes.putAll(buildStartTimeFromFatBuild);
- runHistCompactedDao.setBuildsStartTimeAsync(srvId, buildStartTimeFromFatBuild);
+ runHistCompactedDao.setBuildsStartTime(srvId, buildStartTimeFromFatBuild);
}
Set<Integer> buildInScope = buildIds.stream().filter(
@@ -235,6 +237,14 @@ public class HistoryCollector {
Map<Integer, SuiteInvocation> addl = addSuiteInvocationsToHistory(srvId, missedBuildsIds, normalizedBaseBranch);
suiteRunHist.putAll(addl);
+
+ /*
+ Map<Integer, SuiteInvocation> reloaded = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
+
+ addl.keySet().forEach((k) -> {
+ Preconditions.checkState( reloaded.containsKey(k));
+ });
+ */
}
SuiteHistory sumary = new SuiteHistory();
@@ -268,7 +278,8 @@ public class HistoryCollector {
Map<Integer, SuiteInvocation> suiteRunHist = new HashMap<>();
int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
- System.err.println("GET ALL: " + missedBuildsIds.size());
+ System.err.println(Thread.currentThread().getName() + ": GET ALL: " + missedBuildsIds.size());
+
Iterables.partition(missedBuildsIds, 32 * 10).forEach(
chunk -> {
fatBuildDao.getAllFatBuilds(srvId, chunk).forEach((buildCacheKey, fatBuildCompacted) -> {
@@ -298,7 +309,7 @@ public class HistoryCollector {
+ " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": added " +
suiteRunHist.size() + " invocations from " + missedBuildsIds.size() + " builds checked");
- histDao.putAllAsync(srvId, suiteRunHist);
+ histDao.putAll(srvId, suiteRunHist);
return suiteRunHist;
}
@@ -308,8 +319,8 @@ public class HistoryCollector {
* @param buildTypeId Build type id.
* @param normalizedBaseBranch Normalized base branch.
*/
- public IRunHistory getSuiteRunHist(int srvId, int buildTypeId, int normalizedBaseBranch) {
- return getSuiteHist(srvId, buildTypeId, normalizedBaseBranch).getSuiteHist();
+ public ISuiteRunHistory getSuiteRunHist(int srvId, int buildTypeId, int normalizedBaseBranch) {
+ return getSuiteHist(srvId, buildTypeId, normalizedBaseBranch);
}
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java
new file mode 100644
index 0000000..38dff1c
--- /dev/null
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.tcignited.history;
+
+public interface ISuiteRunHistory {
+ IRunHistory self();
+ IRunHistory getTestRunHist(int testName);
+}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
index a5bb997..ffc0c47 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
@@ -260,7 +260,7 @@ public class RunHistCompactedDao {
return res;
}
- public void setBuildsStartTimeAsync(int srvId, Map<Integer, Long> builds) {
+ public void setBuildsStartTime(int srvId, Map<Integer, Long> builds) {
Map<Long, Long> res = new HashMap<>();
builds.forEach((buildId, ts) -> {
@@ -268,6 +268,6 @@ public class RunHistCompactedDao {
res.put(buildIdToCacheKey(srvId, buildId), ts);
});
- buildStartTime.putAllAsync(res);
+ buildStartTime.putAll(res);
}
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
index b8504fa..dd93c5e 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
@@ -48,7 +48,7 @@ public class SuiteInvocationHistoryDao {
private IgniteCache<Long, SuiteInvocation> suiteHistory;
public void init() {
- CacheConfiguration<Long , SuiteInvocation> ccfg = CacheConfigs.getCacheV2Config("suiteHistory");
+ CacheConfiguration<Long , SuiteInvocation> ccfg = CacheConfigs.getCacheV2Config("teamcitySuiteHistory");
ccfg.setExpiryPolicyFactory(AccessedExpiryPolicy.factoryOf(new Duration(HOURS, 12)));
ccfg.setEagerTtl(true);
@@ -77,11 +77,11 @@ public class SuiteInvocationHistoryDao {
}
@AutoProfiling
- public void putAllAsync(int srvId, Map<Integer, SuiteInvocation> addl) {
+ public void putAll(int srvId, Map<Integer, SuiteInvocation> addl) {
Map<Long, SuiteInvocation> data = new HashMap<>();
addl.forEach((k, v) -> data.put(BuildRefDao.buildIdToCacheKey(srvId, k), v));
- suiteHistory.putAllAsync(data);
+ suiteHistory.putAll(data);
}
}