You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/06/21 17:08:07 UTC
[ignite-teamcity-bot] branch test-hist-performance updated (0895500
-> c802af7)
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a change to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git.
from 0895500 Trusted tests & suite history performance fixes: border implemented for skipping too old builds
new 6f84638 Trusted tests & suite history performance fixes: suite history implementation
new c802af7 Trusted tests & suite history performance fixes: tests fix, disable of call to old hist services
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../ignite/ci/tcbot/issue/IssueDetector.java | 18 ++-
.../ci/teamcity/ignited/TeamcityIgnitedMock.java | 10 +-
.../tcbot/engine/chain/BuildChainProcessor.java | 6 +-
.../ignite/tcbot/engine/chain/MultBuildRunCtx.java | 16 ++-
.../ignite/tcbot/engine/pr/PrChainsProcessor.java | 2 +-
.../apache/ignite/tcbot/engine/ui/DsSuiteUi.java | 17 ++-
.../apache/ignite/tcignited/ITeamcityIgnited.java | 2 +
.../ignite/tcignited/TeamcityIgnitedImpl.java | 19 ++-
.../apache/ignite/tcignited/build/FatBuildDao.java | 2 +-
.../tcignited/build/ProactiveFatBuildSync.java | 3 +-
.../ignite/tcignited/build/SuiteHistory.java | 15 ++-
.../ignite/tcignited/history/HistoryCollector.java | 137 +++++++++------------
.../tcignited/history/RunHistCompactedDao.java | 4 +-
.../ignite/tcignited/history/SuiteInvocation.java | 8 +-
14 files changed, 142 insertions(+), 117 deletions(-)
[ignite-teamcity-bot] 02/02: Trusted tests & suite history
performance fixes: tests fix, disable of call to old hist services
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit c802af79427d4472ad1708b820ca9d42bad38e63
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Jun 21 20:05:55 2019 +0300
Trusted tests & suite history performance fixes: tests fix, disable of call to old hist services
---
.../apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java | 10 ++++------
.../apache/ignite/tcbot/engine/chain/BuildChainProcessor.java | 5 +++--
.../org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java | 5 ++++-
.../java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java | 3 ++-
.../apache/ignite/tcignited/build/ProactiveFatBuildSync.java | 3 ++-
5 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
index 4f9eacd..c744616 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
@@ -90,10 +90,10 @@ public class TeamcityIgnitedMock {
.collect(Collectors.toList());
});
- when(tcIgnited.getTestRunHist(anyString(), anyString()))
+ when(tcIgnited.getTestRunHist(anyInt(), anyInt(), anyInt()))
.thenAnswer((inv) -> {
- final String name = inv.getArgument(0);
- final String branch = inv.getArgument(1);
+ final Integer tstName = inv.getArgument(0);
+ final Integer branchId = inv.getArgument(2);
// System.out.println("Search history " + name + " in " + branch + ": " );
if (histCache.isEmpty()) {
@@ -103,11 +103,9 @@ public class TeamcityIgnitedMock {
}
}
- final Integer tstName = c.getStringIdIfPresent(name);
if (tstName == null)
return null;
- final Integer branchId = c.getStringIdIfPresent(branch);
if (branchId == null)
return null;
@@ -115,7 +113,7 @@ public class TeamcityIgnitedMock {
final RunHistCompacted runHistCompacted = histCache.get(key);
- System.out.println("Test history " + name + " in " + branch + " => " + runHistCompacted);
+ System.out.println("Test history " + c.getStringFromId(tstName) + " in " + c.getStringFromId(branchId) + " => " + runHistCompacted);
return runHistCompacted;
});
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
index f8ee591..144c6ef 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
@@ -144,6 +144,8 @@ public class BuildChainProcessor {
if (entryPoints.isEmpty())
return new FullChainRunCtx(Build.createFakeStub());
+ Integer failRateBranchId = compactor.getStringIdIfPresent(RunHistSync.normalizeBranch(failRateBranch));
+
Map<Integer, Future<FatBuildCompacted>> builds = loadAllBuildsInChains(entryPoints, mode, tcIgn);
Map<String, List<Future<FatBuildCompacted>>> freshRebuilds = new ConcurrentHashMap<>();
@@ -185,8 +187,7 @@ public class BuildChainProcessor {
});
Function<MultBuildRunCtx, Float> function = ctx -> {
- IRunHistory runStat = ctx.history(tcIgn,
- compactor.getStringIdIfPresent(RunHistSync.normalizeBranch(failRateBranch)));
+ IRunHistory runStat = ctx.history(tcIgn, failRateBranchId);
if (runStat == null)
return 0f;
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
index 6d45206..232b88d 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
@@ -644,7 +644,10 @@ public class MultBuildRunCtx implements ISuiteResults {
* @param baseBranchId Base branch id.
*/
public IRunHistory history(ITeamcityIgnited tcIgn, Integer baseBranchId) {
+ if (baseBranchId == null)
+ return null;
+
return historyCacheMap.computeIfAbsent(baseBranchId,
- (k)-> tcIgn.getSuiteRunHist(buildTypeIdId(), k));
+ (k) -> tcIgn.getSuiteRunHist(buildTypeIdId(), k));
}
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index 81a74c8..f15ecab 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -511,7 +511,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
// schedule find missing later
fatBuildSync.ensureActualizationRequested(srvCode, conn);
- runHistSync.invokeLaterFindMissingHistory(srvCode);
+ //todo remove unused code
+ // runHistSync.invokeLaterFindMissingHistory(srvCode);
}
/** {@inheritDoc} */
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java
index 3e01c05..8ad06a7 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java
@@ -315,7 +315,8 @@ public class ProactiveFatBuildSync {
buildRefDao.save(srvIdMask, refCompacted);
- runHistSync.saveToHistoryLater(srvCode, savedVer);
+ //todo remove unused code
+ // runHistSync.saveToHistoryLater(srvCode, savedVer);
return savedVer;
}
[ignite-teamcity-bot] 01/02: Trusted tests & suite history
performance fixes: suite history implementation
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 6f8463859e49ae6bbdfda5e0290a946923415f31
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Jun 21 19:25:58 2019 +0300
Trusted tests & suite history performance fixes: suite history implementation
---
.../ignite/ci/tcbot/issue/IssueDetector.java | 18 ++-
.../tcbot/engine/chain/BuildChainProcessor.java | 5 +-
.../ignite/tcbot/engine/chain/MultBuildRunCtx.java | 13 +-
.../ignite/tcbot/engine/pr/PrChainsProcessor.java | 2 +-
.../apache/ignite/tcbot/engine/ui/DsSuiteUi.java | 17 ++-
.../apache/ignite/tcignited/ITeamcityIgnited.java | 2 +
.../ignite/tcignited/TeamcityIgnitedImpl.java | 16 ++-
.../apache/ignite/tcignited/build/FatBuildDao.java | 2 +-
.../ignite/tcignited/build/SuiteHistory.java | 15 ++-
.../ignite/tcignited/history/HistoryCollector.java | 137 +++++++++------------
.../tcignited/history/RunHistCompactedDao.java | 4 +-
.../ignite/tcignited/history/SuiteInvocation.java | 8 +-
12 files changed, 130 insertions(+), 109 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
index 0dc8d98..ead0965 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
@@ -261,13 +261,14 @@ public class IssueDetector {
final String trackedBranch = res.getTrackedBranch();
+ String suiteId = suiteCurrentStatus.suiteId;
for (DsTestFailureUi testFailure : suiteCurrentStatus.testFailures) {
- if (registerTestFailIssues(tcIgnited, srvCode, normalizeBranch, testFailure, trackedBranch,
+ if (registerTestFailIssues(tcIgnited, srvCode, suiteId, normalizeBranch, testFailure, trackedBranch,
suiteCurrentStatus.tags))
newIssues++;
}
- if (registerSuiteFailIssues(tcIgnited, srvCode, normalizeBranch, suiteCurrentStatus, trackedBranch))
+ if (registerSuiteFailIssues(tcIgnited, srvCode, suiteId, normalizeBranch, suiteCurrentStatus, trackedBranch))
newIssues++;
}
}
@@ -286,13 +287,15 @@ public class IssueDetector {
*/
private boolean registerSuiteFailIssues(ITeamcityIgnited tcIgnited,
String srvCode,
+ String suiteId,
String normalizeBranch,
DsSuiteUi suiteFailure,
String trackedBranch) {
- String suiteId = suiteFailure.suiteId;
+ Integer btId = compactor.getStringIdIfPresent(suiteId);
+ Integer brNormId = compactor.getStringIdIfPresent(normalizeBranch);
- IRunHistory runStat = tcIgnited.getSuiteRunHist(suiteId, normalizeBranch);
+ IRunHistory runStat = tcIgnited.getSuiteRunHist(btId, brNormId);
if (runStat == null)
return false;
@@ -361,14 +364,17 @@ public class IssueDetector {
private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited,
String srvCode,
+ String suiteId,
String normalizeBranch,
DsTestFailureUi testFailure,
String trackedBranch,
@Nonnull Set<String> suiteTags) {
String name = testFailure.name;
- Integer tname = compactor.getStringIdIfPresent(name);
+ int tname = compactor.getStringId(name);
+ Integer btId = compactor.getStringIdIfPresent(suiteId);
+ Integer brNormId = compactor.getStringIdIfPresent(normalizeBranch);
- IRunHistory runStat = tcIgnited.getTestRunHist(name, normalizeBranch);
+ IRunHistory runStat = tcIgnited.getTestRunHist(tname, btId, brNormId);
if (runStat == null)
return false;
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
index ff88d77..f8ee591 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
@@ -185,9 +185,8 @@ public class BuildChainProcessor {
});
Function<MultBuildRunCtx, Float> function = ctx -> {
-
- //todo cache RunStat instance into suite context to compare
- IRunHistory runStat = tcIgn.getSuiteRunHist(ctx.suiteId(), RunHistSync.normalizeBranch(failRateBranch));
+ IRunHistory runStat = ctx.history(tcIgn,
+ compactor.getStringIdIfPresent(RunHistSync.normalizeBranch(failRateBranch)));
if (runStat == null)
return 0f;
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
index 5fb0b2d..6d45206 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/MultBuildRunCtx.java
@@ -27,6 +27,7 @@ import java.util.Objects;
import java.util.Optional;
import java.util.OptionalDouble;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
@@ -34,7 +35,6 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
@@ -67,6 +67,8 @@ public class MultBuildRunCtx implements ISuiteResults {
/** Builds: Single execution. */
private List<SingleBuildRunCtx> builds = new CopyOnWriteArrayList<>();
+ private java.util.Map<Integer, IRunHistory> historyCacheMap = new ConcurrentHashMap<>();
+
public void addBuild(SingleBuildRunCtx ctx) {
builds.add(ctx);
}
@@ -636,4 +638,13 @@ public class MultBuildRunCtx implements ISuiteResults {
public Optional<SingleBuildRunCtx> firstBuild() {
return builds.stream().findFirst();
}
+
+ /**
+ * @param tcIgn Tc ign.
+ * @param baseBranchId Base branch id.
+ */
+ public IRunHistory history(ITeamcityIgnited tcIgn, Integer baseBranchId) {
+ return historyCacheMap.computeIfAbsent(baseBranchId,
+ (k)-> tcIgn.getSuiteRunHist(buildTypeIdId(), k));
+ }
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
index 0e93b57..2da9f5e 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
@@ -259,7 +259,7 @@ public class PrChainsProcessor {
.map((ctx) -> {
String normalizedBaseBranch = RunHistSync.normalizeBranch(baseBranch);
Integer baseBranchId = compactor.getStringIdIfPresent(normalizedBaseBranch);
- IRunHistory statInBaseBranch = tcIgnited.getSuiteRunHist(ctx.suiteId(), normalizedBaseBranch);
+ IRunHistory statInBaseBranch = ctx.history(tcIgnited, baseBranchId);
Integer suiteId = compactor.getStringIdIfPresent(ctx.suiteId()); // can be inlined
String suiteComment = ctx.getPossibleBlockerComment(compactor, statInBaseBranch, tcIgnited.config());
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
index 8a1b003..c823fd9 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsSuiteUi.java
@@ -168,7 +168,7 @@ public class DsSuiteUi extends DsHistoryStatUi {
String curBranchNormalized = normalizeBranch(suite.branchName());
Integer curBranchId = compactor.getStringIdIfPresent(curBranchNormalized);
- IRunHistory baseBranchHist = initSuiteStat(tcIgnited, failRateNormalizedBranch, curBranchNormalized, suite.suiteId());
+ IRunHistory baseBranchHist = initSuiteStat(tcIgnited, baseBranchId, curBranchId, suite);
Set<String> collect = suite.lastChangeUsers().collect(Collectors.toSet());
@@ -264,13 +264,13 @@ public class DsSuiteUi extends DsHistoryStatUi {
}
private IRunHistory initSuiteStat(ITeamcityIgnited tcIgnited,
- String failRateNormalizedBranch,
- String curBranchNormalized,
- String suiteId) {
- if (Strings.isNullOrEmpty(suiteId))
+ Integer failRateNormalizedBranch,
+ Integer curBranchNormalized,
+ MultBuildRunCtx suite) {
+ if (suite.buildTypeIdId() == null)
return null;
- final IRunHistory statInBaseBranch = tcIgnited.getSuiteRunHist(suiteId, failRateNormalizedBranch);
+ IRunHistory statInBaseBranch = suite.history(tcIgnited, failRateNormalizedBranch);
if (statInBaseBranch != null) {
failures = statInBaseBranch.getFailuresCount();
@@ -289,9 +289,8 @@ public class DsSuiteUi extends DsHistoryStatUi {
}
IRunHistory latestRunsSrc = null;
- if (!failRateNormalizedBranch.equals(curBranchNormalized)) {
-
- final IRunHistory statForStripe = tcIgnited.getSuiteRunHist(suiteId, curBranchNormalized);
+ if (!Objects.equals(failRateNormalizedBranch, curBranchNormalized)) {
+ IRunHistory statForStripe = suite.history(tcIgnited, curBranchNormalized);
latestRunsSrc = statForStripe;
latestRuns = statForStripe != null ? statForStripe.getLatestRunResults() : null;
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
index f19a4ad..7cd950e 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
@@ -203,8 +203,10 @@ public interface ITeamcityIgnited {
@Deprecated
@Nullable public IRunHistory getTestRunHist(String testName, @Nullable String branch);
+ @Deprecated
@Nullable public IRunHistory getSuiteRunHist(String suiteId, @Nullable String branch);
+ @Nullable public IRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch);
/**
* V.3.0 run history implementation based on scan of fat builds.
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index 18ea52b..81a74c8 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -16,10 +16,8 @@
*/
package org.apache.ignite.tcignited;
-import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import java.io.File;
-import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -34,7 +32,6 @@ import java.util.OptionalInt;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Function;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -51,7 +48,6 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.GuavaCached;
@@ -72,7 +68,6 @@ import org.apache.ignite.tcignited.history.RunHistSync;
import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
import org.apache.ignite.tcignited.mute.MuteDao;
import org.apache.ignite.tcignited.mute.MuteSync;
-import org.apache.ignite.tcservice.ITeamcity;
import org.apache.ignite.tcservice.ITeamcityConn;
import org.apache.ignite.tcservice.model.agent.Agent;
import org.apache.ignite.tcservice.model.conf.Project;
@@ -441,6 +436,17 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
/** {@inheritDoc} */
+ @Nullable @Override public IRunHistory getSuiteRunHist(@Nullable Integer buildTypeId, @Nullable Integer normalizedBaseBranch) {
+ if (buildTypeId == null || normalizedBaseBranch == null)
+ return null;
+
+ if (buildTypeId < 0 || normalizedBaseBranch < 0)
+ return null;
+
+ return histCollector.getSuiteRunHist(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public IRunHistory getTestRunHist(int testName, @Nullable Integer buildTypeId,
@Nullable Integer normalizedBaseBranch) {
if (buildTypeId == null || normalizedBaseBranch == null)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index 36f355c..b40e204 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -134,7 +134,7 @@ public class FatBuildDao {
public void putFatBuild(int srvIdMaskHigh, int buildId, FatBuildCompacted newBuild) {
buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
- histCollector.invalidateHistoryInMem(srvIdMaskHigh, Stream.of(newBuild));
+ histCollector.invalidateHistoryInMem(srvIdMaskHigh, newBuild);
}
public static int[] extractChangeIds(@Nonnull ChangesList changesList) {
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
index 394b75d..cfd24fe 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
@@ -20,7 +20,6 @@ package org.apache.ignite.tcignited.build;
import java.util.HashMap;
import java.util.Map;
-import java.util.TreeMap;
import org.apache.ignite.Ignite;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
@@ -33,7 +32,9 @@ import org.apache.ignite.tcignited.history.SuiteInvocation;
*/
public class SuiteHistory {
/** Tests history: Test name ID->RunHistory */
- Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
+ private Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
+
+ private RunHistCompacted suiteHist = new RunHistCompacted();
public int size(Ignite ignite) {
BinaryObjectExImpl binary = ignite.binary().toBinary(this);
@@ -51,4 +52,14 @@ public class SuiteHistory {
public void addTestInvocation(Integer tName, Invocation invocation) {
getOrAddTestsHistory(tName).innerAddInvocation(invocation);
}
+
+ public void addSuiteInvocation(SuiteInvocation suiteInv) {
+ suiteInv.tests().forEach(this::addTestInvocation);
+
+ suiteHist.innerAddInvocation(suiteInv.suiteInvocation());
+ }
+
+ public RunHistCompacted getSuiteHist() {
+ return suiteHist;
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
index b204eae..4a62534 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiPredicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.inject.Inject;
@@ -47,11 +48,16 @@ import org.apache.ignite.tcignited.build.SuiteHistory;
import org.apache.ignite.tcignited.buildref.BranchEquivalence;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
*/
public class HistoryCollector {
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(HistoryCollector.class);
+
/** History DAO. */
@Inject private SuiteInvocationHistoryDao histDao;
@@ -84,16 +90,25 @@ public class HistoryCollector {
.softValues()
.build();
+ /** Biggest build ID, which out of history scope (MAX days + 2). */
+ private final ConcurrentMap<Integer, AtomicInteger> biggestBuildIdOutOfHistoryScope = new ConcurrentHashMap<>();
+
/**
* @param srvIdMaskHigh Server id mask to be placed at high bits in the key.
* @param testName Test name.
* @param buildTypeId Suite (Build type) id.
* @param normalizedBaseBranch Branch name.
*/
- @AutoProfiling
public IRunHistory getTestRunHist(int srvIdMaskHigh, int testName, int buildTypeId,
int normalizedBaseBranch) {
+ SuiteHistory hist = getSuiteHist(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
+
+ return hist.getTestRunHist(testName);
+ }
+
+ @AutoProfiling
+ protected SuiteHistory getSuiteHist(int srvIdMaskHigh, int buildTypeId, int normalizedBaseBranch) {
RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
SuiteHistory hist;
@@ -105,11 +120,9 @@ public class HistoryCollector {
throw ExceptionUtil.propagateException(e);
}
- return hist.getTestRunHist(testName);
+ return hist;
}
- ConcurrentMap<Integer, AtomicInteger> biggestBuildIdOutOfHistoryScope = new ConcurrentHashMap<>();
-
/**
* Latest actual Build ids supplier. This supplier should handle all equivalent branches in
* it.
@@ -123,14 +136,14 @@ public class HistoryCollector {
int srvId, int buildTypeId, int normalizedBaseBranch, Set<Integer> knownBuilds) {
String btId = compactor.getStringFromId(buildTypeId);
String branchId = compactor.getStringFromId(normalizedBaseBranch);
- List<BuildRefCompacted> compacted = buildRefDao.getAllBuildsCompacted(srvId, btId,
+ List<BuildRefCompacted> bRefsList = buildRefDao.getAllBuildsCompacted(srvId, btId,
branchEquivalence.branchForQuery(branchId));
AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId);
int outOfScopeBuildId = biggestIdOutOfScope == null ? -1 : biggestIdOutOfScope.get();
long curTs = System.currentTimeMillis();
- Set<Integer> buildIds = compacted.stream()
+ Set<Integer> buildIds = bRefsList.stream()
.filter(b -> b.id() > outOfScopeBuildId)
.filter(this::applicableForHistory)
.map(BuildRefCompacted::id)
@@ -140,31 +153,28 @@ public class HistoryCollector {
+ compactor.getStringFromId(buildTypeId)
+ " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + buildIds.size() + " builds" );
- Map<Integer, Long> buildStartTimeFromSeparatedCache = getStartTimeFromSpecialCache(srvId, buildIds);
+ Map<Integer, Long> buildStartTimes = getStartTimeFromSpecialCache(srvId, buildIds);
- HashSet<Integer> notFoundKeys = new HashSet<>(buildIds);
- notFoundKeys.removeAll(buildStartTimeFromSeparatedCache.keySet());
+ Set<Integer> notFoundKeys = new HashSet<>(buildIds);
+ notFoundKeys.removeAll(buildStartTimes.keySet());
if (!notFoundKeys.isEmpty()) {
Map<Integer, Long> buildStartTimeFromFatBuild = getStartTimeFromFatBuild(srvId, notFoundKeys);
- buildStartTimeFromSeparatedCache.putAll(buildStartTimeFromFatBuild);
+ buildStartTimes.putAll(buildStartTimeFromFatBuild);
- runHistCompactedDao.setBuildsStartTime(srvId, buildStartTimeFromFatBuild);
+ runHistCompactedDao.setBuildsStartTimeAsync(srvId, buildStartTimeFromFatBuild);
}
- //todo filter queued, cancelled and so on
+
Set<Integer> buildInScope = buildIds.stream().filter(
bId -> {
- //todo getAll
- //todo getStartTime From FatBuild
- Long startTime = buildStartTimeFromSeparatedCache.get(bId);
- //todo cache in runHistCompactedDao.getBuildStartTime(srvId, bRef.id());
+ Long startTime = buildStartTimes.get(bId);
if (startTime == null)
return false;
long ageInDays = Duration.ofMillis(curTs - startTime).toDays();
- if(ageInDays>InvocationData.MAX_DAYS + 2) {
+ if (ageInDays > InvocationData.MAX_DAYS + 2) {
AtomicInteger integer = biggestBuildIdOutOfHistoryScope.computeIfAbsent(srvId,
s -> {
AtomicInteger atomicInteger = new AtomicInteger();
@@ -175,7 +185,7 @@ public class HistoryCollector {
int newBorder = integer.accumulateAndGet(bId, Math::max);
if (newBorder == bId)
- System.err.println(" New border for server was set " + bId);
+ logger.info("History Collector: New border for server was set " + bId);
}
return ageInDays < InvocationData.MAX_DAYS;
@@ -183,7 +193,7 @@ public class HistoryCollector {
).collect(Collectors.toSet());
System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " +
- buildInScope.size() + " from " + buildIds.size());
+ buildInScope.size() + " from " + bRefsList.size());
return buildInScope;
}
@@ -206,7 +216,7 @@ public class HistoryCollector {
}
@AutoProfiling
- public SuiteHistory loadSuiteHistory(int srvId,
+ protected SuiteHistory loadSuiteHistory(int srvId,
int buildTypeId,
int normalizedBaseBranch) {
Map<Integer, SuiteInvocation> suiteRunHist = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
@@ -229,45 +239,50 @@ public class HistoryCollector {
SuiteHistory sumary = new SuiteHistory();
- suiteRunHist.forEach((buildId, suiteInv) -> suiteInv.tests().forEach(sumary::addTestInvocation));
+ suiteRunHist.forEach((buildId, suiteInv) -> sumary.addSuiteInvocation(suiteInv));
- System.err.println("***** History for suite "
- + compactor.getStringFromId(buildTypeId)
- + " branch" + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
- sumary.size(igniteProvider.get()) + " bytes");
+ if (logger.isDebugEnabled()) {
+ logger.debug("***** History for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch" + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
+ sumary.size(igniteProvider.get()) + " bytes");
+ }
return sumary;
}
- public void invalidateHistoryInMem(int srvId, Stream<BuildRefCompacted> stream) {
- Iterable<RunHistKey> objects =
- stream
- .map(b -> new RunHistKey(srvId, b.buildTypeId(), b.branchName()))
- .collect(Collectors.toSet());
+ /**
+ * @param srvId Server id.
+ * @param b Build ref to invalidate.
+ */
+ public void invalidateHistoryInMem(int srvId, BuildRefCompacted b) {
+ RunHistKey inv = new RunHistKey(srvId, b.buildTypeId(), b.branchName());
- runHistInMemCache.invalidateAll(objects);
+ runHistInMemCache.invalidate(inv);
}
@AutoProfiling
- public Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId,
+ protected Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId,
HashSet<Integer> missedBuildsIds, int normalizedBaseBranch) {
Map<Integer, SuiteInvocation> suiteRunHist = new HashMap<>();
int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
System.err.println("GET ALL: " + missedBuildsIds.size());
- Iterables.partition(missedBuildsIds, 32*10).forEach(
- chunk->{
+ Iterables.partition(missedBuildsIds, 32 * 10).forEach(
+ chunk -> {
fatBuildDao.getAllFatBuilds(srvId, chunk).forEach((buildCacheKey, fatBuildCompacted) -> {
- if(!applicableForHistory(fatBuildCompacted))
+ if (!applicableForHistory(fatBuildCompacted))
return;
- SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, (k, v) -> false);
+ BiPredicate<Integer, Integer> paramsFilter = (k, v) -> false;
+
+ SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, paramsFilter);
Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
tests.forEach(
testCompacted -> {
- Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
+ Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, paramsFilter, successStatusStrId);
sinv.addTest(testCompacted.testName(), invocation);
}
@@ -288,45 +303,13 @@ public class HistoryCollector {
return suiteRunHist;
}
- /*
- @AutoProfiling
- protected SuiteHistory calcSuiteHistory(int srvIdMaskHigh, Set<Integer> buildIds) {
- Set<Long> cacheKeys = buildsIdsToCacheKeys(srvIdMaskHigh, buildIds);
-
- int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
-
- CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> processor = new FatBuildDao.HistoryCollectProcessor(successStatusStrId);
-
- Map<Long, EntryProcessorResult<Map<Integer, Invocation>>> map = buildsCache.invokeAll(cacheKeys, processor);
-
- SuiteHistory hist = new SuiteHistory();
-
- map.values().forEach(
- res -> {
- if (res == null)
- return;
-
- Map<Integer, Invocation> invocationMap = res.get();
-
- if (invocationMap == null)
- return;
-
- invocationMap.forEach((k, v) -> {
- RunHistCompacted compacted = hist.testsHistory.computeIfAbsent(k,
- k_ -> new RunHistCompacted());
-
- compacted.innerAddInvocation(v);
- });
-
- }
- );
-
- System.err.println("Suite history: tests in scope "
- + hist.testsHistory.size()
- + " for " +buildIds.size() + " builds checked"
- + " size " + hist.size(igniteProvider.get()));
-
- return hist;
+ /**
+ * @param srvId Server id.
+ * @param buildTypeId Build type id.
+ * @param normalizedBaseBranch Normalized base branch.
+ */
+ public IRunHistory getSuiteRunHist(int srvId, int buildTypeId, int normalizedBaseBranch) {
+ return getSuiteHist(srvId, buildTypeId, normalizedBaseBranch).getSuiteHist();
}
- */
+
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
index ffc0c47..a5bb997 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
@@ -260,7 +260,7 @@ public class RunHistCompactedDao {
return res;
}
- public void setBuildsStartTime(int srvId, Map<Integer, Long> builds) {
+ public void setBuildsStartTimeAsync(int srvId, Map<Integer, Long> builds) {
Map<Long, Long> res = new HashMap<>();
builds.forEach((buildId, ts) -> {
@@ -268,6 +268,6 @@ public class RunHistCompactedDao {
res.put(buildIdToCacheKey(srvId, buildId), ts);
});
- buildStartTime.putAll(res);
+ buildStartTime.putAllAsync(res);
}
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
index 8bc8d95..7342af7 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
@@ -46,9 +46,9 @@ public class SuiteInvocation {
@QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 2)})
private int normalizedBranchName;
- Invocation suite;
+ private Invocation suite;
- java.util.Map<Integer, Invocation> tests = new HashMap<>();
+ private Map<Integer, Invocation> tests = new HashMap<>();
Long buildStartTime;
@@ -70,4 +70,8 @@ public class SuiteInvocation {
public Map<Integer, Invocation> tests() {
return Collections.unmodifiableMap(tests);
}
+
+ public Invocation suiteInvocation() {
+ return suite;
+ }
}