You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/11/29 18:11:03 UTC
[ignite-teamcity-bot] branch master updated: IGNITE-9542 provide
separated base/current branch history for PR page. - Fixes #84.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 77728c2 IGNITE-9542 provide separated base/current branch history for PR page. - Fixes #84.
77728c2 is described below
commit 77728c2162f83eef92c48814acaedfedc4df0590
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Nov 29 21:10:49 2018 +0300
IGNITE-9542 provide separated base/current branch history for PR page. - Fixes #84.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
---
.../java/org/apache/ignite/ci/ITcAnalytics.java | 2 +
.../main/java/org/apache/ignite/ci/ITeamcity.java | 1 +
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 22 +-
.../main/java/org/apache/ignite/ci/TcHelper.java | 2 +-
...ITestFailures.java => IMultTestOccurrence.java} | 5 +-
.../apache/ignite/ci/analysis/MultBuildRunCtx.java | 6 +-
.../org/apache/ignite/ci/analysis/RunStat.java | 66 +---
.../ignite/ci/analysis/TestCompactedMult.java | 2 +-
.../java/org/apache/ignite/ci/db/DbMigrations.java | 23 ++
.../java/org/apache/ignite/ci/db/TcHelperDb.java | 24 +-
.../ignite/ci/di/scheduler/TcBotScheduler.java | 32 +-
.../ci/github/ignited/GitHubConnIgnitedImpl.java | 15 +-
.../org/apache/ignite/ci/issue/IssueDetector.java | 7 +-
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 20 +-
.../ignite/ci/tcbot/chain/PrChainsProcessor.java | 16 +-
.../tcbot/chain/TrackedBranchChainsProcessor.java | 2 +-
.../tcbot/visa/TcBotTriggerAndSignOffService.java | 2 +-
.../ci/teamcity/ignited/BuildRefCompacted.java | 4 +
.../ignite/ci/teamcity/ignited/IRunHistory.java | 64 ++++
.../ignited/IRunStat.java} | 34 ++-
.../ci/teamcity/ignited/ITeamcityIgnited.java | 16 +-
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 56 +++-
.../ci/teamcity/ignited/TeamcityIgnitedModule.java | 7 +
.../buildcondition/BuildConditionCompacted.java | 1 -
.../ignited/buildref/BuildRefSync.java} | 19 +-
.../ignited/fatbuild/FatBuildCompacted.java | 43 ++-
.../ci/teamcity/ignited/fatbuild/FatBuildDao.java | 2 +-
.../ignited/fatbuild/ProactiveFatBuildSync.java | 22 +-
.../ignited/fatbuild/ProblemCompacted.java | 9 +
.../teamcity/ignited/fatbuild/TestCompacted.java | 20 ++
.../ci/teamcity/ignited/runhist/Invocation.java | 86 ++++++
.../teamcity/ignited/runhist/InvocationData.java | 186 ++++++++++++
.../teamcity/ignited/runhist/RunHistCompacted.java | 123 ++++++++
.../ignited/runhist/RunHistCompactedDao.java | 217 +++++++++++++
.../ci/teamcity/ignited/runhist/RunHistKey.java | 77 +++++
.../ci/teamcity/ignited/runhist/RunHistSync.java | 336 +++++++++++++++++++++
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
.../model/current/ChainAtServerCurrentStatus.java | 31 +-
.../ci/web/model/current/SuiteCurrentStatus.java | 75 +++--
.../ignite/ci/web/model/current/TestFailure.java | 34 ++-
.../ignite/ci/web/model/hist/TestHistory.java | 9 +-
.../ignite/ci/web/rest/GetChainResultsAsHtml.java | 6 +-
.../ci/web/rest/build/GetBuildTestFailures.java | 11 +-
.../ci/tcbot/chain/BuildChainProcessorTest.java | 6 +-
.../ci/tcbot/chain/PrChainsProcessorTest.java | 209 ++++++++++---
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 111 ++++++-
.../ci/teamcity/ignited/TeamcityIgnitedMock.java | 64 +++-
.../ignited/TeamcityIgnitedProviderMock.java | 3 +-
48 files changed, 1830 insertions(+), 300 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITcAnalytics.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITcAnalytics.java
index 9320610..87a198a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITcAnalytics.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITcAnalytics.java
@@ -34,11 +34,13 @@ public interface ITcAnalytics {
*
* @return map from suite ID to its run statistics
*/
+ @Deprecated
Function<SuiteInBranch, RunStat> getBuildFailureRunStatProvider();
/**
* @return map from test full name (suite: suite.test) and its branch to its run statistics
*/
+ @Deprecated
Function<TestInBranch, RunStat> getTestRunStatProvider();
List<RunStat> topFailingSuite(int cnt);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index bab8d36..62332e1 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -44,6 +44,7 @@ import java.util.concurrent.ExecutorService;
public interface ITeamcity extends ITeamcityConn {
String DEFAULT = "<default>";
+ String REFS_HEADS_MASTER = "refs/heads/master";
@Deprecated
long DEFAULT_BUILDS_COUNT = 1000;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index b8dc136..8ebc749 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -58,7 +58,7 @@ import java.util.function.Predicate;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
-import static org.apache.ignite.ci.tcbot.chain.BuildChainProcessor.normalizeBranch;
+import static org.apache.ignite.ci.teamcity.ignited.runhist.RunHistSync.normalizeBranch;
/**
* Apache Ignite based cache over teamcity responses (REST caches).
@@ -70,6 +70,7 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
//V2 caches, 32 parts (V1 caches were 1024 parts)
@Deprecated
private static final String TESTS_RUN_STAT = "testsRunStat";
+ @Deprecated
private static final String CALCULATED_STATISTIC = "calculatedStatistic";
private static final String LOG_CHECK_RESULT = "logCheckResult";
@@ -319,7 +320,7 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
}
@NotNull private SuiteInBranch keyForBuild(Build loaded) {
- return new SuiteInBranch(loaded.suiteId(), normalizeBranch(loaded));
+ return new SuiteInBranch(loaded.suiteId(), normalizeBranch(loaded.branchName));
}
private Build realLoadBuild(String href1) {
@@ -349,9 +350,12 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
return teamcity.host();
}
+ @Deprecated
private void registerCriticalBuildProblemInStat(BuildRef build, ProblemOccurrences problems) {
boolean criticalFail = problems.getProblemsNonNull().stream().anyMatch(occurrence ->
- occurrence.isExecutionTimeout() || occurrence.isJvmCrash() || occurrence.isFailureOnMetric()
+ occurrence.isExecutionTimeout()
+ || occurrence.isJvmCrash()
+ || occurrence.isFailureOnMetric()
|| occurrence.isCompilationError());
String suiteId = build.suiteId();
@@ -361,7 +365,7 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
return;
if (buildId != null && !Strings.isNullOrEmpty(suiteId)) {
- SuiteInBranch key = new SuiteInBranch(suiteId, normalizeBranch(build));
+ SuiteInBranch key = new SuiteInBranch(suiteId, normalizeBranch(build.branchName));
buildsFailureRunStatCache().invoke(key, (entry, arguments) -> {
SuiteInBranch suiteInBranch = entry.getKey();
@@ -409,10 +413,12 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
.map(Cache.Entry::getValue);
}
+ @Deprecated
private IgniteCache<TestInBranch, RunStat> testRunStatCache() {
return getOrCreateCacheV2(ignCacheNme(TESTS_RUN_STAT));
}
+ @Deprecated
private IgniteCache<Integer, Boolean> calculatedStatistic() {
return getOrCreateCacheV2(ignCacheNme(CALCULATED_STATISTIC));
}
@@ -446,6 +452,7 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
return getOrCreateCacheV2(ignCacheNme(LOG_CHECK_RESULT));
}
+ @Deprecated
private void addTestOccurrenceToStat(TestOccurrence next, String normalizedBranch, Boolean changesExist) {
String name = next.getName();
if (Strings.isNullOrEmpty(name))
@@ -517,8 +524,11 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
if (calculatedStatistic().containsKey(ctx.buildId()))
return;
- for (TestOccurrence testOccurrence : ctx.getTests())
- addTestOccurrenceToStat(testOccurrence, normalizeBranch(ctx.getBranch()), !ctx.getChanges().isEmpty());
+ for (TestOccurrence testOccurrence : ctx.getTests()) {
+ String branch = normalizeBranch(ctx.getBranch());
+
+ addTestOccurrenceToStat(testOccurrence, branch, !ctx.getChanges().isEmpty());
+ }
calculatedStatistic().put(ctx.buildId(), true);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/TcHelper.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/TcHelper.java
index 63cf693..9832d2a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/TcHelper.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/TcHelper.java
@@ -173,7 +173,7 @@ public class TcHelper implements ITcHelper, IJiraIntegration {
JiraCommentResponse res;
try {
- List<SuiteCurrentStatus> suitesStatuses = prChainsProcessor.getSuitesStatuses(buildTypeId, build.branchName, srvId, prov);
+ List<SuiteCurrentStatus> suitesStatuses = prChainsProcessor.getBlockersSuitesStatuses(buildTypeId, build.branchName, srvId, prov);
if (suitesStatuses == null)
return new Visa("JIRA wasn't commented - no finished builds to analyze.");
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailures.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/IMultTestOccurrence.java
similarity index 90%
copy from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailures.java
copy to ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/IMultTestOccurrence.java
index 7cfc21c..ec7cce2 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailures.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/IMultTestOccurrence.java
@@ -17,13 +17,12 @@
package org.apache.ignite.ci.analysis;
-import java.util.stream.Stream;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
/**
- * Multiple test ocurrence
+ * Multiple test occurrence. For single build context - max 1 failure
*/
-public interface ITestFailures {
+public interface IMultTestOccurrence {
String getName();
boolean isInvestigated();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
index 2e55f62..2a942de 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
@@ -255,8 +255,8 @@ public class MultBuildRunCtx implements ISuiteResults {
return CollectionUtil.top(logSizeBytes.entrySet().stream(), 3 ,comparing).stream();
}
- public Stream<? extends ITestFailures> getTopLongRunning() {
- Comparator<ITestFailures> comparing = Comparator.comparing(ITestFailures::getAvgDurationMs);
+ public Stream<? extends IMultTestOccurrence> getTopLongRunning() {
+ Comparator<IMultTestOccurrence> comparing = Comparator.comparing(IMultTestOccurrence::getAvgDurationMs);
Map<Integer, TestCompactedMult> res = new HashMap<>();
@@ -267,7 +267,7 @@ public class MultBuildRunCtx implements ISuiteResults {
return CollectionUtil.top(res.values().stream(), 3, comparing).stream();
}
- public List<ITestFailures> getFailedTests() {
+ public List<IMultTestOccurrence> getFailedTests() {
Map<Integer, TestCompactedMult> res = new HashMap<>();
builds.forEach(singleBuildRunCtx -> {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/RunStat.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/RunStat.java
index 7bed35a..72d17b6 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/RunStat.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/RunStat.java
@@ -29,6 +29,7 @@ import org.apache.ignite.ci.db.Persisted;
import org.apache.ignite.ci.issue.EventTemplate;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
+import org.apache.ignite.ci.teamcity.ignited.IRunHistory;
import org.jetbrains.annotations.NotNull;
import static org.apache.ignite.ci.analysis.RunStat.RunStatus.RES_CRITICAL_FAILURE;
@@ -40,7 +41,7 @@ import static org.apache.ignite.ci.analysis.RunStat.RunStatus.RES_OK;
* Test or Build run statistics.
*/
@Persisted
-public class RunStat {
+public class RunStat implements IRunHistory {
public static final int MAX_LATEST_RUNS = 100;
/**
@@ -174,82 +175,37 @@ public class RunStat {
return name;
}
- public float getFailRateAllHist() {
- if (runs == 0)
- return 1.0f;
- return 1.0f * failures / runs;
- }
- public int getFailuresAllHist() {
+ @Override public int getFailuresAllHist() {
return failures;
}
- public int getRunsAllHist() {
+ @Override public int getRunsAllHist() {
return runs;
}
- /**
- * @return
- */
- public String getFailPercentAllHistPrintable() {
- return getPercentPrintable(getFailRateAllHist() * 100.0f);
- }
- /**
- * @return float representing fail rate
- */
- public float getFailRate() {
- int runs = getRunsCount();
-
- if (runs == 0)
- return 1.0f;
- return 1.0f * getFailuresCount() / runs;
- }
- /**
- * @return float representing fail rate
- */
- public float getCriticalFailRate() {
- int runs = getRunsCount();
-
- if (runs == 0)
- return 1.0f;
-
- return 1.0f * getCriticalFailuresCount() / runs;
- }
-
- public int getFailuresCount() {
+ @Override public int getFailuresCount() {
if (latestRuns == null)
return 0;
return (int)latestRuns.values().stream().filter(res -> res.status != RES_OK).count();
}
- public int getCriticalFailuresCount() {
+ @Override public int getCriticalFailuresCount() {
if (latestRuns == null)
return 0;
return (int)latestRuns.values().stream().filter(res -> res.status == RES_CRITICAL_FAILURE).count();
}
- public int getRunsCount() {
+ @Override public int getRunsCount() {
return latestRuns == null ? 0 : latestRuns.size();
}
- public String getFailPercentPrintable() {
- return getPercentPrintable(getFailRate() * 100.0f);
- }
-
- public String getCriticalFailPercentPrintable() {
- return getPercentPrintable(getCriticalFailRate() * 100.0f);
- }
-
- private static String getPercentPrintable(float percent) {
- return String.format("%.1f", percent).replace(".", ",");
- }
-
public long getAverageDurationMs() {
if (runsWithDuration == 0)
return 0;
@@ -301,14 +257,14 @@ public class RunStat {
* @return
*/
@Nullable
- public List<Integer> getLatestRunResults() {
+ @Override public List<Integer> getLatestRunResults() {
if (latestRuns == null)
return Collections.emptyList();
return latestRuns.values().stream().map(info -> info.status.code).collect(Collectors.toList());
}
- private int[] concatArr(int[] arr1, int[] arr2) {
+ private static int[] concatArr(int[] arr1, int[] arr2) {
int[] arr1and2 = new int[arr1.length + arr2.length];
System.arraycopy(arr1, 0, arr1and2, 0, arr1.length);
System.arraycopy(arr2, 0, arr1and2, arr1.length, arr2.length);
@@ -354,7 +310,7 @@ public class RunStat {
}
@Nullable
- private TestId checkTemplateAtPos(int[] template, int centralEvtBuild, List<Map.Entry<TestId, RunInfo>> histAsArr,
+ private static TestId checkTemplateAtPos(int[] template, int centralEvtBuild, List<Map.Entry<TestId, RunInfo>> histAsArr,
int idx) {
for (int tIdx = 0; tIdx < template.length; tIdx++) {
RunInfo curStatus = histAsArr.get(idx + tIdx).getValue();
@@ -381,7 +337,7 @@ public class RunStat {
}
@Nullable
- public String getFlakyComments() {
+ @Override public String getFlakyComments() {
if (latestRuns == null)
return null;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/TestCompactedMult.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/TestCompactedMult.java
index e76dcf7..2b80f40 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/TestCompactedMult.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/TestCompactedMult.java
@@ -25,7 +25,7 @@ import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
-public class TestCompactedMult implements ITestFailures {
+public class TestCompactedMult implements IMultTestOccurrence {
private final List<TestCompacted> occurrences = new ArrayList<>();
private IStringCompactor compactor;
private long avgDuration = -1;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
index 443195e..cec2753 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
@@ -115,6 +115,18 @@ public class DbMigrations {
String TESTS = "tests";
String TEST_REFS = "testRefs";
String ISSUES_USAGES_LIST = "issuesUsagesList";
+
+ /** Cache name.*/
+ public static final String TEST_HIST_CACHE_NAME = "testRunHistV0";
+ public static final String TEST_HIST_CACHE_NAME2 ="teamcityTestRunHistV0";
+
+ /** Build Start time Cache name. */
+ public static final String BUILD_START_TIME_CACHE_NAME = "buildStartTimeV0";
+ public static final String BUILD_START_TIME_CACHE_NAME2 = "teamcityBuildStartTimeV0";
+
+ /** Cache name.*/
+ public static final String SUITE_HIST_CACHE_NAME = "teamcitySuiteRunHistV0";
+
}
private final Ignite ignite;
@@ -396,6 +408,13 @@ public class DbMigrations {
applyDestroyIgnCacheMigration(Old.TEST_REFS);
applyDestroyIgnCacheMigration(Old.ISSUES_USAGES_LIST);
+
+ applyDestroyCacheMigration(Old.SUITE_HIST_CACHE_NAME);
+ applyDestroyCacheMigration(Old.BUILD_START_TIME_CACHE_NAME);
+ applyDestroyCacheMigration(Old.TEST_HIST_CACHE_NAME);
+ applyDestroyCacheMigration(Old.TEST_HIST_CACHE_NAME2);
+ applyDestroyCacheMigration(Old.BUILD_START_TIME_CACHE_NAME);
+ applyDestroyCacheMigration(Old.BUILD_START_TIME_CACHE_NAME2);
}
private void applyDestroyIgnCacheMigration(String cacheName) {
@@ -403,6 +422,10 @@ public class DbMigrations {
applyDestroyCacheMigration(cacheName, ignCacheNme);
}
+ private void applyDestroyCacheMigration(String cacheNme) {
+ applyDestroyCacheMigration(cacheNme, cacheNme);
+ }
+
private void applyDestroyCacheMigration(String dispCacheName, String cacheNme) {
applyMigration("destroy-" + dispCacheName, () -> {
IgniteCache<Object, Object> cache = ignite.cache(cacheNme);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/TcHelperDb.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/TcHelperDb.java
index 23f7e62..140859e 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/TcHelperDb.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/TcHelperDb.java
@@ -136,42 +136,62 @@ public class TcHelperDb {
}
+ @NotNull
+ public static <K, V> CacheConfiguration<K, V> getCache8PartsConfig(String name) {
+ CacheConfiguration<K, V> ccfg = new CacheConfiguration<>(name);
+
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 8));
+
+ return ccfg;
+ }
+
public static class LocalOnlyTcpDiscoveryIpFinder implements TcpDiscoveryIpFinder {
+ /** Port. */
private int port;
+ /**
+ * @param port Port.
+ */
public LocalOnlyTcpDiscoveryIpFinder(int port) {
this.port = port;
}
+ /** {@inheritDoc} */
@Override public void onSpiContextInitialized(IgniteSpiContext spiCtx) throws IgniteSpiException {
}
+ /** {@inheritDoc} */
@Override public void onSpiContextDestroyed() {
}
- @Override
- public void initializeLocalAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException {
+ /** {@inheritDoc} */
+ @Override public void initializeLocalAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException {
}
+ /** {@inheritDoc} */
@Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException {
return Collections.singletonList(new InetSocketAddress("localhost", port));
}
+ /** {@inheritDoc} */
@Override public boolean isShared() {
return false;
}
+ /** {@inheritDoc} */
@Override public void registerAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException {
}
+ /** {@inheritDoc} */
@Override public void unregisterAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException {
}
+ /** {@inheritDoc} */
@Override public void close() {
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
index c931b08..959c840 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ci.di.scheduler;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import org.apache.ignite.ci.di.MonitoredTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,44 +29,55 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
class TcBotScheduler implements IScheduler {
- public static final int POOL_SIZE = 10;
+ public static final int POOL_SIZE = 16;
/** Logger. */
private static final Logger logger = LoggerFactory.getLogger(TcBotScheduler.class);
+ private ThreadFactory threadFactory = Executors.defaultThreadFactory();
+
+ private final ThreadFactory threadFactory1 = r -> {
+ Thread thread = threadFactory.newThread(r);
+
+ thread.setName("tc-scheduler-" + thread.getName());
+
+ return thread;
+ };
+
/** Executor service. */
- private volatile ScheduledExecutorService executorSvc = Executors.newScheduledThreadPool(POOL_SIZE);
+ private volatile ScheduledExecutorService executorSvc = Executors.newScheduledThreadPool(POOL_SIZE, threadFactory1);
/** Submit named task checker guard. */
private AtomicBoolean tickGuard = new AtomicBoolean();
+ /** Named tasks. */
private final ConcurrentMap<String, NamedTask> namedTasks = new ConcurrentHashMap<>();
+ /** {@inheritDoc} */
@Override public void invokeLater(Runnable cmd, long delay, TimeUnit unit) {
service().schedule(cmd, delay, unit);
}
+ /** {@inheritDoc} */
@Override public void sheduleNamed(String fullName, Runnable cmd, long queitPeriod, TimeUnit unit) {
NamedTask task = namedTasks.computeIfAbsent(fullName, NamedTask::new);
task.sheduleWithQuitePeriod(cmd, queitPeriod, unit);
if (tickGuard.compareAndSet(false, true)) {
- for (int i = 0; i < POOL_SIZE; i++) {
- int threadNo = i;
-
+ for (int threadId = 0; threadId < POOL_SIZE; threadId++) {
+ String threadNme = "Runner " + Strings.padStart(Integer.toString(threadId), 2, '0');
int period = 15000 + ThreadLocalRandom.current().nextInt(10000);
- service().scheduleAtFixedRate(() -> checkNamedTasks(threadNo), 0, period, TimeUnit.MILLISECONDS);
+ service().scheduleAtFixedRate(() -> checkNamedTasks(threadNme), 0, period, TimeUnit.MILLISECONDS);
}
}
}
/**
- *
- * @param threadNo
+ * @param threadNme Runner name to be used in display.
*/
@SuppressWarnings({"UnusedReturnValue", "WeakerAccess"})
- @MonitoredTask(name = "Scheduled, runner", nameExtArgIndex = 0)
- protected String checkNamedTasks(int threadNo) {
+ @MonitoredTask(name = "Scheduled", nameExtArgIndex = 0)
+ protected String checkNamedTasks(String threadNme) {
AtomicInteger run = new AtomicInteger();
List<Throwable> problems = new ArrayList<>();
namedTasks.forEach((s, task) -> {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
index b672223..799a769 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
@@ -29,14 +29,12 @@ import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.ci.db.TcHelperDb;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.di.MonitoredTask;
import org.apache.ignite.ci.di.scheduler.IScheduler;
import org.apache.ignite.ci.github.PullRequest;
import org.apache.ignite.ci.github.pure.IGitHubConnection;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.jetbrains.annotations.NotNull;
/**
*
@@ -69,16 +67,7 @@ class GitHubConnIgnitedImpl implements IGitHubConnIgnited {
srvIdMaskHigh = Math.abs(srvId.hashCode());
- prCache = igniteProvider.get().getOrCreateCache(getCache8PartsConfig(GIT_HUB_PR));
- }
-
- @NotNull
- public static <K, V> CacheConfiguration<K, V> getCache8PartsConfig(String name) {
- CacheConfiguration<K, V> ccfg = new CacheConfiguration<>(name);
-
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 8));
-
- return ccfg;
+ prCache = igniteProvider.get().getOrCreateCache(TcHelperDb.getCache8PartsConfig(GIT_HUB_PR));
}
/** {@inheritDoc} */
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueDetector.java
index b11c87c..fbe54eb 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueDetector.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/issue/IssueDetector.java
@@ -50,10 +50,6 @@ import org.apache.ignite.ci.di.MonitoredTask;
import org.apache.ignite.ci.jobs.CheckQueueJob;
import org.apache.ignite.ci.mail.EmailSender;
import org.apache.ignite.ci.mail.SlackSender;
-import org.apache.ignite.ci.tcmodel.changes.Change;
-import org.apache.ignite.ci.tcmodel.changes.ChangeRef;
-import org.apache.ignite.ci.tcmodel.changes.ChangesList;
-import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.user.ICredentialsProv;
import org.apache.ignite.ci.user.TcHelperUser;
import org.apache.ignite.ci.user.UserAndSessionsStorage;
@@ -65,8 +61,7 @@ import org.apache.ignite.ci.web.rest.parms.FullQueryParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static com.google.common.base.Strings.isNullOrEmpty;
-import static org.apache.ignite.ci.tcbot.chain.BuildChainProcessor.normalizeBranch;
+import static org.apache.ignite.ci.teamcity.ignited.runhist.RunHistSync.normalizeBranch;
/**
*
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index 315cef7..a24340b 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -38,7 +38,6 @@ import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.inject.Inject;
import org.apache.ignite.ci.IAnalyticsEnabledTeamcity;
-import org.apache.ignite.ci.ITeamcity;
import org.apache.ignite.ci.analysis.FullChainRunCtx;
import org.apache.ignite.ci.analysis.MultBuildRunCtx;
import org.apache.ignite.ci.analysis.RunStat;
@@ -54,6 +53,7 @@ import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.teamcity.ignited.SyncMode;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistSync;
import org.apache.ignite.ci.util.FutureUtil;
import org.apache.ignite.ci.web.TcUpdatePool;
import org.apache.ignite.ci.web.model.long_running.LRTest;
@@ -209,7 +209,7 @@ public class BuildChainProcessor {
});
Function<MultBuildRunCtx, Float> function = ctx -> {
- SuiteInBranch key = new SuiteInBranch(ctx.suiteId(), normalizeBranch(failRateBranch));
+ SuiteInBranch key = new SuiteInBranch(ctx.suiteId(), RunHistSync.normalizeBranch(failRateBranch));
//todo place RunStat into suite context to compare
RunStat runStat = teamcity.getBuildFailureRunStatProvider().apply(key);
@@ -346,10 +346,6 @@ public class BuildChainProcessor {
throw new UnsupportedOperationException("invalid mode " + includeLatestRebuild);
}
- @NotNull private static String getBranchOrDefault(@Nullable String branchName) {
- return branchName == null ? ITeamcity.DEFAULT : branchName;
- }
-
@SuppressWarnings("WeakerAccess")
@AutoProfiling
protected void fillBuildCounts(MultBuildRunCtx outCtx,
@@ -388,17 +384,9 @@ public class BuildChainProcessor {
}
}
+ @Deprecated
@NotNull public static String normalizeBranch(@NotNull final BuildRef build) {
- return normalizeBranch(build.branchName);
- }
-
- @NotNull public static String normalizeBranch(@Nullable String branchName) {
- String branch = getBranchOrDefault(branchName);
-
- if ("refs/heads/master".equals(branch))
- return ITeamcity.DEFAULT;
-
- return branch;
+ return RunHistSync.normalizeBranch(build.branchName);
}
/**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java
index 8495c41..d5e36db 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java
@@ -40,12 +40,10 @@ import org.apache.ignite.ci.web.model.current.SuiteCurrentStatus;
import org.apache.ignite.ci.web.model.current.TestFailure;
import org.apache.ignite.ci.web.model.current.TestFailuresSummary;
import org.apache.ignite.ci.web.rest.parms.FullQueryParams;
-import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import javax.inject.Inject;
import java.util.List;
-import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
/**
@@ -146,7 +144,7 @@ public class PrChainsProcessor {
runningUpdates.addAndGet(cnt0);
//fail rate reference is always default (master)
- chainStatus.initFromContext(teamcity, ctx, teamcity, baseBranch);
+ chainStatus.initFromContext(tcIgnited, teamcity, ctx, teamcity, baseBranch);
}
res.addChainOnServer(chainStatus);
@@ -163,17 +161,15 @@ public class PrChainsProcessor {
* @param prov Credentials.
* @return List of suites with possible blockers.
*/
- @Nullable public List<SuiteCurrentStatus> getSuitesStatuses(String buildTypeId,
+ @Nullable public List<SuiteCurrentStatus> getBlockersSuitesStatuses(String buildTypeId,
String branchForTc,
String srvId,
ICredentialsProv prov) {
- SyncMode queued = SyncMode.RELOAD_QUEUED;
-
- return getSuitesStatuses(buildTypeId, branchForTc, srvId, prov, queued);
+ return getBlockersSuitesStatuses(buildTypeId, branchForTc, srvId, prov, SyncMode.RELOAD_QUEUED);
}
- @Nullable public List<SuiteCurrentStatus> getSuitesStatuses(String buildTypeId, String branchForTc, String srvId,
+ @Nullable public List<SuiteCurrentStatus> getBlockersSuitesStatuses(String buildTypeId, String branchForTc, String srvId,
ICredentialsProv prov, SyncMode queued) {
List<SuiteCurrentStatus> res = new ArrayList<>();
@@ -187,7 +183,7 @@ public class PrChainsProcessor {
return null;
for (ChainAtServerCurrentStatus server : summary.servers) {
- Map<String, List<SuiteCurrentStatus>> fails = findFailures(server);
+ Map<String, List<SuiteCurrentStatus>> fails = findBlockerFailures(server);
fails.forEach((k, v) -> res.addAll(v));
}
@@ -199,7 +195,7 @@ public class PrChainsProcessor {
* @param srv Server.
* @return Failures for given server.
*/
- private Map<String, List<SuiteCurrentStatus>> findFailures(ChainAtServerCurrentStatus srv) {
+ private Map<String, List<SuiteCurrentStatus>> findBlockerFailures(ChainAtServerCurrentStatus srv) {
Map<String, List<SuiteCurrentStatus>> fails = new LinkedHashMap<>();
for (SuiteCurrentStatus suite : srv.suites) {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchChainsProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchChainsProcessor.java
index be329aa..da4188b 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchChainsProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchChainsProcessor.java
@@ -108,7 +108,7 @@ public class TrackedBranchChainsProcessor {
if (cnt > 0)
runningUpdates.addAndGet(cnt);
- chainStatus.initFromContext(teamcity, ctx, teamcity, baseBranchTc);
+ chainStatus.initFromContext(tcIgnited, teamcity, ctx, teamcity, baseBranchTc);
return chainStatus;
})
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java
index 7a18342..bd3d77b 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java
@@ -465,7 +465,7 @@ public class TcBotTriggerAndSignOffService {
CurrentVisaStatus status = new CurrentVisaStatus();
List<SuiteCurrentStatus> suitesStatuses
- = prChainsProcessor.getSuitesStatuses(buildTypeId, tcBranch, srvId, prov, SyncMode.NONE);
+ = prChainsProcessor.getBlockersSuitesStatuses(buildTypeId, tcBranch, srvId, prov, SyncMode.NONE);
if (suitesStatuses == null)
return status;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
index f6ccf03..ada9317 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
@@ -191,6 +191,10 @@ public class BuildRefCompacted {
return compactor.getStringId(STATE_QUEUED) == state();
}
+ public boolean isSuccess(IStringCompactor compactor) {
+ return compactor.getStringId(STATUS_SUCCESS) == status();
+ }
+
@Override public String toString() {
return MoreObjects.toStringHelper(this)
.add("id", id)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/IRunHistory.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/IRunHistory.java
new file mode 100644
index 0000000..091d8f7
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/IRunHistory.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.teamcity.ignited;
+
+import javax.annotation.Nullable;
+import java.util.List;
+
+/**
+ * Test or Build run statistics.
+ */
+public interface IRunHistory extends IRunStat {
+ public int getFailuresAllHist();
+ public int getRunsAllHist();
+
+ public default float getFailRateAllHist() {
+ if (getRunsAllHist() == 0)
+ return 0.0f;
+
+ return 1.0f * getFailuresAllHist() / getRunsAllHist();
+ }
+
+ public default String getFailPercentAllHistPrintable() {
+ return IRunStat.getPercentPrintable(getFailRateAllHist() * 100.0f);
+ }
+
+
+ @Nullable
+ List<Integer> getLatestRunResults();
+
+ String getFlakyComments();
+
+ public default String getCriticalFailPercentPrintable() {
+ return IRunStat.getPercentPrintable(getCriticalFailRate() * 100.0f);
+ }
+
+ /**
+ * @return float representing fail rate
+ */
+ public default float getCriticalFailRate() {
+ int runs = getRunsCount();
+
+ if (runs == 0)
+ return 1.0f;
+
+ return 1.0f * getCriticalFailuresCount() / runs;
+ }
+
+
+ public int getCriticalFailuresCount();
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailures.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/IRunStat.java
similarity index 55%
copy from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailures.java
copy to ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/IRunStat.java
index 7cfc21c..34ddf36 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailures.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/IRunStat.java
@@ -14,23 +14,33 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.ignite.ci.analysis;
-
-import java.util.stream.Stream;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
+package org.apache.ignite.ci.teamcity.ignited;
/**
- * Multiple test ocurrence
+ * Abstract runs statistics,
*/
-public interface ITestFailures {
- String getName();
+public interface IRunStat {
+ public int getRunsCount();
+ public int getFailuresCount();
+
+ /**
+ * @return fail rate as float.
+ */
+ public default float getFailRate() {
+ int runs = getRunsCount();
+
+ if (runs == 0)
+ return 0.0f;
- boolean isInvestigated();
+ return 1.0f * getFailuresCount() / runs;
+ }
- public int failuresCount();
+ public default String getFailPercentPrintable() {
+ return getPercentPrintable(getFailRate() * 100.0f);
+ }
- public long getAvgDurationMs();
- Iterable<TestOccurrenceFull> getOccurrences();
+ public static String getPercentPrintable(float percent) {
+ return String.format("%.1f", percent).replace(".", ",");
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
index 97414a3..bc3788c 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
@@ -20,10 +20,12 @@ import java.util.Collection;
import java.util.Date;
import java.util.List;
import javax.annotation.Nullable;
+import org.apache.ignite.ci.analysis.SuiteInBranch;
+import org.apache.ignite.ci.analysis.TestInBranch;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.teamcity.ignited.buildcondition.BuildCondition;
-import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeCompacted;
+import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.jetbrains.annotations.NotNull;
@@ -107,8 +109,6 @@ public interface ITeamcityIgnited {
return getFatBuild(id, SyncMode.RELOAD_QUEUED);
}
-
-
/**
* @param id Id.
* @param mode Refresh mode.
@@ -154,4 +154,14 @@ public interface ITeamcityIgnited {
* @return BuildType compacted.
*/
public BuildTypeCompacted getBuildType(String buildTypeId);
+
+ @Nullable public IRunHistory getTestRunHist(TestInBranch testInBranch);
+
+ @Nullable public IRunHistory getSuiteRunHist(SuiteInBranch branch);
+
+ /**
+ * @param suiteBuildTypeId Suite id.
+ * @return run statistics of recent runls on all branches.
+ */
+ @Nullable public IRunStat getSuiteRunStatAllBranches(String suiteBuildTypeId);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 01702cf..df11962 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -21,6 +21,8 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.ignite.ci.ITeamcity;
+import org.apache.ignite.ci.analysis.SuiteInBranch;
+import org.apache.ignite.ci.analysis.TestInBranch;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.di.MonitoredTask;
import org.apache.ignite.ci.di.cache.GuavaCached;
@@ -41,6 +43,8 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompactedDao;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistSync;
import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
@@ -94,7 +98,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Changes DAO. */
@Inject private ChangeDao changesDao;
- /** Changes DAO. */
+ /** Changes sync class. */
@Inject private ChangeSync changeSync;
/** BuildType reference DAO. */
@@ -106,7 +110,13 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** BuildType DAO. */
@Inject private BuildTypeSync buildTypeSync;
- /** Changes DAO. */
+ /** Run history DAO. */
+ @Inject private RunHistCompactedDao runHistCompactedDao;
+
+ /** Run history sync. */
+ @Inject private RunHistSync runHistSync;
+
+ /** Strings compactor. */
@Inject private IStringCompactor compactor;
/** Server ID mask for cache Entries. */
@@ -121,6 +131,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
buildConditionDao.init();
fatBuildDao.init();
changesDao.init();
+ runHistCompactedDao.init();
}
@NotNull
@@ -325,6 +336,26 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
/** {@inheritDoc} */
+ @Nullable
+ @Override public IRunHistory getTestRunHist(TestInBranch testInBranch) {
+ return runHistCompactedDao.getTestRunHist(srvIdMaskHigh, testInBranch.name, testInBranch.branch);
+ }
+
+ /** {@inheritDoc} */
+ @Nullable
+ @Override public IRunHistory getSuiteRunHist(SuiteInBranch suiteInBranch) {
+ return runHistCompactedDao.getSuiteRunHist(srvIdMaskHigh, suiteInBranch.getSuiteId(), suiteInBranch.branch);
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public IRunStat getSuiteRunStatAllBranches(String suiteBuildTypeId) {
+ return runHistCompactedDao.getSuiteRunStatAllBranches(srvIdMaskHigh, suiteBuildTypeId);
+ }
+
+ /**
+ * @param branchName Branch name.
+ */
+ /** {@inheritDoc} */
@Override public List<String> getCompositeBuildTypesIdsSortedByBuildNumberCounter(String projectId) {
return buildTypeSync.getCompositeBuildTypesIdsSortedByBuildNumberCounter(srvIdMaskHigh, projectId, conn);
}
@@ -346,13 +377,18 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
public List<String> branchForQuery(@Nullable String branchName) {
if (ITeamcity.DEFAULT.equals(branchName))
- return Lists.newArrayList(branchName, "refs/heads/master", "master");
+ return Lists.newArrayList(branchName, ITeamcity.REFS_HEADS_MASTER, "master");
else
return Collections.singletonList(branchName);
}
public void ensureActualizeRequested() {
scheduler.sheduleNamed(taskName("actualizeRecentBuildRefs"), this::actualizeRecentBuildRefs, 2, TimeUnit.MINUTES);
+
+ // schedule find missing later
+ buildSync.invokeLaterFindMissingByBuildRef(srvNme, conn);
+
+ runHistSync.invokeLaterFindMissingHistory(srvNme);
}
/** {@inheritDoc} */
@@ -418,11 +454,14 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
FatBuildCompacted savedVer = buildSync.loadBuild(conn, buildId, existingBuild, mode);
- //build was modified, probably we need also to update reference accordindly
- if (savedVer != null)
- buildRefDao.save(srvIdMaskHigh, new BuildRefCompacted(savedVer));
+ //build was modified, probably we need also to update reference accordingly
+ if (savedVer == null)
+ return existingBuild;
+
+ buildRefDao.save(srvIdMaskHigh, new BuildRefCompacted(savedVer));
+ runHistSync.saveToHistoryLater(srvNme, savedVer);
- return savedVer == null ? existingBuild : savedVer;
+ return savedVer;
}
protected FatBuildCompacted getFatBuildFromIgnite(int buildId) {
@@ -461,9 +500,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
*
*/
void actualizeRecentBuildRefs() {
- // schedule find missing later
- buildSync.invokeLaterFindMissingByBuildRef(srvNme, conn);
-
List<BuildRefCompacted> running = buildRefDao.getQueuedAndRunning(srvIdMaskHigh);
Set<Integer> paginateUntil = new HashSet<>();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
index 690f87f..b1d294f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ci.teamcity.ignited;
import com.google.inject.AbstractModule;
import com.google.inject.internal.SingletonScope;
import org.apache.ignite.ci.teamcity.ignited.buildcondition.BuildConditionDao;
+import org.apache.ignite.ci.teamcity.ignited.buildref.BuildRefSync;
import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeRefDao;
import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeSync;
import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeDao;
@@ -26,6 +27,8 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProactiveFatBuildSync;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompactedDao;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistSync;
import org.apache.ignite.ci.teamcity.pure.ITeamcityHttpConnection;
import org.apache.ignite.ci.teamcity.restcached.TcRestCachedModule;
import org.jetbrains.annotations.Nullable;
@@ -41,6 +44,7 @@ public class TeamcityIgnitedModule extends AbstractModule {
@Override protected void configure() {
bind(ITeamcityIgnitedProvider.class).to(TcIgnitedCachingProvider.class).in(new SingletonScope());
bind(BuildRefDao.class).in(new SingletonScope());
+ bind(BuildRefSync.class).in(new SingletonScope());
bind(BuildConditionDao.class).in(new SingletonScope());
bind(FatBuildDao.class).in(new SingletonScope());
bind(ProactiveFatBuildSync.class).in(new SingletonScope());
@@ -49,6 +53,9 @@ public class TeamcityIgnitedModule extends AbstractModule {
bind(BuildTypeRefDao.class).in(new SingletonScope());
bind(BuildTypeDao.class).in(new SingletonScope());
bind(BuildTypeSync.class).in(new SingletonScope());
+ bind(RunHistCompactedDao.class).in(new SingletonScope());
+ bind(RunHistSync.class).in(new SingletonScope());
+
bind(IStringCompactor.class).to(IgniteStringCompactor.class).in(new SingletonScope());
TcRestCachedModule module = new TcRestCachedModule();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildcondition/BuildConditionCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildcondition/BuildConditionCompacted.java
index 93074cd..b4e5713 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildcondition/BuildConditionCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildcondition/BuildConditionCompacted.java
@@ -20,7 +20,6 @@ package org.apache.ignite.ci.teamcity.ignited.buildcondition;
import java.util.Date;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
-//todo move to ignited package, change cache to 8 partitions
public class BuildConditionCompacted {
/** Build id. */
int buildId = -1;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailures.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildref/BuildRefSync.java
similarity index 68%
rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailures.java
rename to ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildref/BuildRefSync.java
index 7cfc21c..fce67c7 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailures.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildref/BuildRefSync.java
@@ -14,23 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.ignite.ci.analysis;
-
-import java.util.stream.Stream;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
+package org.apache.ignite.ci.teamcity.ignited.buildref;
/**
- * Multiple test ocurrence
+ *
*/
-public interface ITestFailures {
- String getName();
-
- boolean isInvestigated();
-
- public int failuresCount();
-
- public long getAvgDurationMs();
-
- Iterable<TestOccurrenceFull> getOccurrences();
+public class BuildRefSync {
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 56f03a0..fd153e9 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -32,6 +32,8 @@ import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
+import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -210,9 +212,8 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
if (snapshotDeps != null) {
List<BuildRef> snapshotDependencies = new ArrayList<>();
- for (int i = 0; i < snapshotDeps.length; i++) {
- int depId = snapshotDeps[i];
+ for (int depId : snapshotDeps) {
BuildRef ref = new BuildRef();
ref.setId(depId);
ref.href = getHrefForId(depId);
@@ -231,7 +232,7 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
* @param compactor Compactor.
* @param page Page.
*/
- public void addTests(IStringCompactor compactor, List<TestOccurrenceFull> page) {
+ public FatBuildCompacted addTests(IStringCompactor compactor, List<TestOccurrenceFull> page) {
for (TestOccurrenceFull next : page) {
TestCompacted compacted = new TestCompacted(compactor, next);
@@ -240,6 +241,8 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
tests.add(compacted);
}
+
+ return this;
}
/**
@@ -289,7 +292,11 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
/** Start date. */
@Nullable public Date getStartDate() {
- return startDate > 0 ? new Date(startDate) : null;
+ return getStartDateTs() > 0 ? new Date(getStartDateTs()) : null;
+ }
+
+ public long getStartDateTs() {
+ return startDate;
}
/** {@inheritDoc} */
@@ -458,4 +465,32 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
.add("changesIds", changesIds)
.toString();
}
+
+ public Invocation toInvocation(IStringCompactor compactor) {
+ boolean success = isSuccess(compactor);
+
+ final Invocation invocation = new Invocation(getId());
+
+ final int failCode ;
+
+ if (success)
+ failCode = InvocationData.OK;
+ else {
+ if (problems()
+ .stream().anyMatch(occurrence ->
+ occurrence.isExecutionTimeout(compactor)
+ || occurrence.isJvmCrash(compactor)
+ || occurrence.isBuildFailureOnMetric(compactor)
+ || occurrence.isCompilationError(compactor)))
+ failCode = InvocationData.CRITICAL_FAILURE;
+ else
+ failCode = InvocationData.FAILURE;
+
+ }
+
+ invocation.status((byte)failCode);
+ invocation.startDate(getStartDateTs());
+ invocation.changesPresent(changes().length > 0 ? 1 : 0);
+ return invocation;
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index aa34f3f..2c4df80 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -113,7 +113,7 @@ public class FatBuildDao {
}
@AutoProfiling
- protected void putFatBuild(long srvIdMaskHigh, int buildId, FatBuildCompacted newBuild) {
+ public void putFatBuild(long srvIdMaskHigh, int buildId, FatBuildCompacted newBuild) {
buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index dc00780..6a63289 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -111,9 +111,10 @@ public class ProactiveFatBuildSync {
@NotNull
public synchronized SyncTask getSyncTask(ITeamcityConn conn) {
- final String serverId = conn.serverId();
- final SyncTask syncTask = buildToLoad.computeIfAbsent(serverId, s -> new SyncTask());
+ final SyncTask syncTask = buildToLoad.computeIfAbsent(conn.serverId(), s -> new SyncTask());
+
syncTask.conn = conn;
+
return syncTask;
}
@@ -148,13 +149,13 @@ public class ProactiveFatBuildSync {
}
/** */
- private void loadFatBuilds(int ldrNo, String serverId) {
+ private void loadFatBuilds(int ldrNo, String srvId) {
Set<Integer> load;
ITeamcityConn conn;
final GridConcurrentHashSet<Integer> loadingBuilds;
synchronized (this) {
- final SyncTask syncTask = buildToLoad.get(serverId);
+ final SyncTask syncTask = buildToLoad.get(srvId);
if (syncTask == null)
return;
@@ -178,7 +179,7 @@ public class ProactiveFatBuildSync {
syncTask.conn = null;
}
- doLoadBuilds(ldrNo, serverId, conn, load, loadingBuilds);
+ doLoadBuilds(ldrNo, srvId, conn, load, loadingBuilds);
}
@SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
@@ -228,13 +229,13 @@ public class ProactiveFatBuildSync {
() -> findMissingBuildsFromBuildRef(srvName, conn), 360, TimeUnit.MINUTES);
}
-
/**
*
- * @param conn
- * @param buildId
- * @param existingBuild
- * @return null if nothing was saved, use existing build
+ * @param conn TC connection to load data
+ * @param buildId build ID (TC identification).
+ * @param existingBuild build from DB.
+ * @return null if nothing was saved, use existing build. Non null value indicates that
+ * new build if it was updated.
*/
@Nullable
public FatBuildCompacted loadBuild(ITeamcityConn conn, int buildId,
@@ -317,6 +318,7 @@ public class ProactiveFatBuildSync {
}
else
build = Build.createFakeStub();
+ //todo here can be situation we have build ref, but don't have a build
} else {
logger.error("Loading build [" + buildId + "] for server [" + srvNme + "] failed:" + e.getMessage(), e);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java
index 7a800e3..c1942e0 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProblemCompacted.java
@@ -124,6 +124,15 @@ public class ProblemCompacted {
return compactor.getStringId(ProblemOccurrence.TC_COMPILATION_ERROR) == type;
}
+
+ /** */
+ public boolean isBuildFailureOnMetric(IStringCompactor compactor) {
+ return compactor.getStringId(ProblemOccurrence.BUILD_FAILURE_ON_METRIC) == type;
+ }
+
+ /**
+ * @param compactor Compactor.
+ */
public boolean isExecutionTimeout(IStringCompactor compactor) {
return compactor.getStringId(ProblemOccurrence.TC_EXECUTION_TIMEOUT) == type;
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
index 7684803..d7b1a99 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/TestCompacted.java
@@ -33,6 +33,8 @@ import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.tcmodel.result.tests.TestRef;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
+import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -373,4 +375,22 @@ public class TestCompacted {
return null;
}
+
+ public Invocation toInvocation(IStringCompactor compactor, FatBuildCompacted build) {
+ final boolean failedTest = isFailedTest(compactor);
+
+ final Invocation invocation = new Invocation(build.getId());
+
+ final int failCode = failedTest
+ ? (isIgnoredTest() || isMutedTest())
+ ? InvocationData.MUTED
+ : InvocationData.FAILURE
+ : InvocationData.OK;
+
+ invocation.status((byte) failCode);
+ invocation.startDate(build.getStartDateTs());
+ invocation.changesPresent(build.changes().length > 0 ? 1 : 0);
+
+ return invocation;
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/Invocation.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/Invocation.java
new file mode 100644
index 0000000..f540c2f
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/Invocation.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.teamcity.ignited.runhist;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+
+public class Invocation {
+ private int buildId;
+ byte status;
+ private byte changePresent;
+ long startDate;
+
+ public Invocation(Integer buildId) {
+ this.buildId = buildId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("buildId", buildId)
+ .add("status", status)
+ .add("changePresent", changePresent)
+ .toString();
+ }
+
+ public void status(int failCode) {
+ Preconditions.checkState(failCode < 128);
+ this.status = (byte) failCode;
+ }
+
+ public void startDate(long startDateTs) {
+ this.startDate = startDateTs;
+ }
+
+ public void changesPresent(int changesPresent) {
+ Preconditions.checkState(changesPresent < 128);
+ this.changePresent = (byte) changesPresent;
+ }
+
+ public boolean isFailure() {
+ return status == InvocationData.FAILURE || status == InvocationData.MUTED;
+ }
+
+ public int buildId() {
+ return buildId;
+ }
+
+ public long startDate() {
+ return startDate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ Invocation that = (Invocation)o;
+ return buildId == that.buildId &&
+ status == that.status &&
+ changePresent == that.changePresent &&
+ startDate == that.startDate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hashCode(buildId, status, changePresent, startDate);
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java
new file mode 100644
index 0000000..7ccc0fa
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.teamcity.ignited.runhist;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
+import java.time.Duration;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.ignite.ci.analysis.RunStat;
+
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.NotNull;
+
+public class InvocationData {
+ public static final int MAX_DAYS = 30;
+ /** Muted. */
+ public static final int MUTED = RunStat.RunStatus.RES_MUTED_FAILURE.getCode();
+ /** Failure. */
+ public static final int FAILURE = RunStat.RunStatus.RES_FAILURE.getCode();
+ /** Ok. */
+ public static final int OK = RunStat.RunStatus.RES_OK.getCode();
+ /** Ok. */
+ public static final int CRITICAL_FAILURE = RunStat.RunStatus.RES_CRITICAL_FAILURE.getCode();
+
+ /**
+ * Runs registered all the times.
+ */
+ private int allHistRuns;
+
+ /**
+ * Failures registered all the times.
+ */
+ private int allHistFailures;
+
+ /** Invocations map from build ID to invocation data. */
+ private Map<Integer, Invocation> invocationMap = new TreeMap<>();
+
+ public int allHistRuns() {
+ return allHistRuns;
+ }
+
+ public boolean addInvocation(Invocation inv) {
+ try {
+ return innerAdd(inv);
+ }
+ finally {
+ removeEldiest();
+ }
+ }
+
+ public boolean innerAdd(Invocation inv) {
+ int build = inv.buildId();
+ if (build < 0)
+ return false;
+
+ if (invocationMap.containsKey(build))
+ return false;
+
+ if (isExpired(inv.startDate))
+ return false;
+
+ Invocation prevVal = invocationMap.putIfAbsent(build, inv);
+
+ final boolean newVal = prevVal == null;
+
+ if (newVal) {
+ allHistRuns++;
+ if (inv.isFailure())
+ allHistFailures++;
+ }
+
+ return newVal;
+ }
+
+ void removeEldiest() {
+ invocationMap.entrySet().removeIf(entries -> isExpired(entries.getValue().startDate));
+ }
+
+ /**
+ * @param startDate Start date.
+ */
+ public static boolean isExpired(long startDate) {
+ return (U.currentTimeMillis() - startDate) > Duration.ofDays(MAX_DAYS).toMillis();
+ }
+
+ /**
+ *
+ */
+ public int allHistFailures() {
+ return allHistFailures;
+ }
+
+ /**
+ *
+ */
+ public int notMutedRunsCount() {
+ return (int)
+ invocations()
+ .filter(invocation -> invocation.status != MUTED)
+ .count();
+ }
+
+ /**
+ *
+ */
+ @NotNull public Stream<Invocation> invocations() {
+ return invocationMap.values()
+ .stream()
+ .filter(this::isActual);
+ }
+
+ /**
+ * @param invocation Invocation.
+ */
+ private boolean isActual(Invocation invocation) {
+ return !isExpired(invocation.startDate);
+ }
+
+ /**
+ *
+ */
+ public int failuresCount() {
+ return (int)invocations().filter(inv -> inv.status == FAILURE || inv.status == CRITICAL_FAILURE).count();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("allHistRuns", allHistRuns)
+ .add("allHistFailures", allHistFailures)
+ .add("invocationMap", invocationMap)
+ .toString();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ InvocationData data = (InvocationData)o;
+ return allHistRuns == data.allHistRuns &&
+ allHistFailures == data.allHistFailures &&
+ Objects.equal(invocationMap, data.invocationMap);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hashCode(allHistRuns, allHistFailures, invocationMap);
+ }
+
+ /**
+ *
+ */
+ public List<Integer> getLatestRuns() {
+ return invocations()
+ .map(i->(int)i.status)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ *
+ */
+ public int criticalFailuresCount() {
+ return (int)invocations().filter(inv -> inv.status == CRITICAL_FAILURE).count();
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
new file mode 100644
index 0000000..23f0fa4
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.teamcity.ignited.runhist;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.ignite.ci.analysis.IVersionedEntity;
+import org.apache.ignite.ci.db.Persisted;
+import org.apache.ignite.ci.teamcity.ignited.IRunHistory;
+
+/**
+ *
+ */
+@Persisted
+public class RunHistCompacted implements IVersionedEntity, IRunHistory {
+ /** Latest version. */
+ private static final int LATEST_VERSION = 1;
+
+ /** Entity fields version. */
+ @SuppressWarnings("FieldCanBeLocal")
+ private short _ver = LATEST_VERSION;
+
+ private InvocationData data = new InvocationData();
+
+ public RunHistCompacted() {}
+
+ public RunHistCompacted(RunHistKey ignored) {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public int version() {
+ return _ver;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int latestVersion() {
+ return LATEST_VERSION;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getRunsCount() {
+ return data.notMutedRunsCount();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getFailuresCount() {
+ return data.failuresCount();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getFailuresAllHist() {
+ return data.allHistFailures();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getRunsAllHist() {
+ return data.allHistRuns();
+ }
+
+ /** {@inheritDoc} */
+ @Nullable
+ @Override public List<Integer> getLatestRunResults() {
+ return data.getLatestRuns();
+ }
+
+ @Override public String getFlakyComments() {
+ return null; //todo implement
+ }
+
+ @Override public int getCriticalFailuresCount() {
+ return data.criticalFailuresCount();
+ }
+
+ /**
+ * @param inv Invocation.
+ * @return if test run is new and is not expired.
+ */
+ public boolean addInvocation(Invocation inv) {
+ return data.addInvocation(inv);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("failRate", getFailPercentPrintable())
+ .add("data", data)
+ .toString();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ RunHistCompacted compacted = (RunHistCompacted)o;
+ return _ver == compacted._ver &&
+ Objects.equal(data, compacted.data);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hashCode(_ver, data);
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
new file mode 100644
index 0000000..e5d86e3
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.teamcity.ignited.runhist;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.Cache;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.ci.db.TcHelperDb;
+import org.apache.ignite.ci.di.AutoProfiling;
+import org.apache.ignite.ci.teamcity.ignited.IRunHistory;
+import org.apache.ignite.ci.teamcity.ignited.IRunStat;
+import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.configuration.CacheConfiguration;
+
+import javax.inject.Inject;
+import javax.inject.Provider;
+import java.util.Collections;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.ci.teamcity.ignited.runhist.RunHistSync.normalizeBranch;
+
+/**
+ *
+ */
+public class RunHistCompactedDao {
+ /** Cache name. */
+ public static final String TEST_HIST_CACHE_NAME = "teamcityTestRunHist";
+
+ /** Build Start time Cache name. */
+ public static final String BUILD_START_TIME_CACHE_NAME = "teamcityBuildStartTime";
+
+ /** Suites history Cache name.*/
+ public static final String SUITE_HIST_CACHE_NAME = "teamcitySuiteRunHist";
+
+
+ /** Ignite provider. */
+ @Inject
+ private Provider<Ignite> igniteProvider;
+
+ /** Test history cache. */
+ private IgniteCache<RunHistKey, RunHistCompacted> testHistCache;
+
+ /** Suite history cache. */
+ private IgniteCache<RunHistKey, RunHistCompacted> suiteHistCacheName;
+
+ /** Build start time. */
+ private IgniteCache<Long, Long> buildStartTime;
+
+ /** Compactor. */
+ @Inject private IStringCompactor compactor;
+
+ /**
+ * Initialize
+ */
+ public void init () {
+ Ignite ignite = igniteProvider.get();
+
+ final CacheConfiguration<RunHistKey, RunHistCompacted> cfg = TcHelperDb.getCacheV2Config(TEST_HIST_CACHE_NAME);
+
+ cfg.setQueryEntities(Collections.singletonList(new QueryEntity(RunHistKey.class, RunHistCompacted.class)));
+
+ testHistCache = ignite.getOrCreateCache(cfg);
+
+ buildStartTime = ignite.getOrCreateCache(TcHelperDb.getCacheV2Config(BUILD_START_TIME_CACHE_NAME));
+
+ final CacheConfiguration<RunHistKey, RunHistCompacted> cfg2 = TcHelperDb.getCache8PartsConfig(SUITE_HIST_CACHE_NAME);
+
+ cfg2.setQueryEntities(Collections.singletonList(new QueryEntity(RunHistKey.class, RunHistCompacted.class)));
+
+ suiteHistCacheName = ignite.getOrCreateCache(cfg2);
+
+ buildStartTime = ignite.getOrCreateCache(TcHelperDb.getCacheV2Config(BUILD_START_TIME_CACHE_NAME));
+ }
+
+ public IRunHistory getTestRunHist(int srvIdMaskHigh, String name, @Nullable String branch) {
+ RunHistKey key = getKey(srvIdMaskHigh, name, branch);
+ if (key == null)
+ return null;
+
+ return testHistCache.get(key);
+ }
+
+ @Nullable public RunHistKey getKey(int srvIdMaskHigh, String testOrSuiteName, @Nullable String branch) {
+ final Integer testName = compactor.getStringIdIfPresent(testOrSuiteName);
+ if (testName == null)
+ return null;
+
+ final Integer branchId = compactor.getStringIdIfPresent(normalizeBranch(branch));
+ if (branchId == null)
+ return null;
+
+ return new RunHistKey(srvIdMaskHigh, testName, branchId);
+ }
+
+ /**
+ * @param srvId Server id mask high.
+ * @param buildId Build id.
+ */
+ public static long buildIdToCacheKey(long srvId, int buildId) {
+ return (long)buildId | srvId << 32;
+ }
+
+ @AutoProfiling
+ public boolean buildWasProcessed(int srvId, int buildId) {
+ return buildStartTime.containsKey(buildIdToCacheKey(srvId, buildId));
+ }
+
+ @AutoProfiling
+ public boolean setBuildProcessed(int srvId, int buildId, long ts) {
+ return buildStartTime.putIfAbsent(buildIdToCacheKey(srvId, buildId), ts);
+ }
+
+ @AutoProfiling
+ public Integer addTestInvocations(RunHistKey histKey, List<Invocation> list) {
+ if (list.isEmpty())
+ return 0;
+
+ return testHistCache.invoke(histKey, RunHistCompactedDao::processEntry, list);
+ }
+
+
+ @AutoProfiling
+ public Integer addSuiteInvocations(RunHistKey histKey, List<Invocation> list) {
+ if (list.isEmpty())
+ return 0;
+
+ return suiteHistCacheName.invoke(histKey, RunHistCompactedDao::processEntry, list);
+ }
+
+ @NotNull public static Integer processEntry(MutableEntry<RunHistKey, RunHistCompacted> entry, Object[] parms) {
+ int cnt = 0;
+
+ RunHistCompacted hist = entry.getValue();
+
+ if (hist == null)
+ hist = new RunHistCompacted(entry.getKey());
+
+ int initHashCode = hist.hashCode();
+
+ List<Invocation> invocationList = (List<Invocation>)parms[0];
+
+ for (Invocation invocation : invocationList) {
+ if (hist.addInvocation(invocation))
+ cnt++;
+ }
+
+ if (cnt > 0 || hist.hashCode() != initHashCode)
+ entry.setValue(hist);
+
+ return cnt;
+ }
+
+ /**
+ * @param srvId Server id.
+ * @param suiteId Suite id.
+ * @param branch Branch.
+ */
+ public IRunHistory getSuiteRunHist(int srvId, String suiteId, @Nullable String branch) {
+ RunHistKey key = getKey(srvId, suiteId, branch);
+ if (key == null)
+ return null;
+
+ return suiteHistCacheName.get(key);
+ }
+
+ public IRunStat getSuiteRunStatAllBranches(int srvIdMaskHigh, String btId) {
+ final Integer testName = compactor.getStringIdIfPresent(btId);
+ if (testName == null)
+ return null;
+
+ AtomicInteger runs = new AtomicInteger();
+ AtomicInteger failures = new AtomicInteger();
+ try (QueryCursor<Cache.Entry<RunHistKey, RunHistCompacted>> qryCursor = suiteHistCacheName.query(
+ new SqlQuery<RunHistKey, RunHistCompacted>(RunHistCompacted.class, "testOrSuiteName = ? and srvId = ?")
+ .setArgs(testName, srvIdMaskHigh))) {
+
+ for (Cache.Entry<RunHistKey, RunHistCompacted> next : qryCursor) {
+ RunHistCompacted val = next.getValue();
+
+ runs.addAndGet(val.getRunsCount());
+ failures.addAndGet(val.getFailuresCount());
+ }
+ }
+
+ return new IRunStat() {
+ @Override public int getRunsCount() {
+ return runs.get();
+ }
+
+ @Override public int getFailuresCount() {
+ return failures.get();
+ }
+ };
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistKey.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistKey.java
new file mode 100644
index 0000000..239e479
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistKey.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.teamcity.ignited.runhist;
+
+import com.google.common.base.Objects;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+
+/**
+ *
+ */
+public class RunHistKey {
+ /** Server ID. */
+ @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "tstAndSrv", order = 1)})
+ private int srvId;
+
+ /** Test name or suite build type ID. */
+ @QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "tstAndSrv", order = 0)})
+ private int testOrSuiteName;
+
+ /** Branch name. */
+ private int branch;
+
+ /**
+ * @param srvId Server id.
+ * @param testName Test or suite name.
+ * @param branchName Branch name.
+ */
+ public RunHistKey(int srvId, int testName, int branchName) {
+ this.srvId = srvId;
+ this.testOrSuiteName = testName;
+ this.branch = branchName;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ RunHistKey histKey = (RunHistKey) o;
+ return srvId == histKey.srvId &&
+ testOrSuiteName == histKey.testOrSuiteName &&
+ branch == histKey.branch;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hashCode(srvId, testOrSuiteName, branch);
+ }
+
+ /**
+ *
+ */
+ public int testNameOrSuite() {
+ return testOrSuiteName;
+ }
+
+ /**
+ *
+ */
+ public int srvId() {
+ return srvId;
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
new file mode 100644
index 0000000..11d43c4
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.teamcity.ignited.runhist;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import javax.annotation.concurrent.GuardedBy;
+import javax.inject.Inject;
+import org.apache.ignite.ci.ITeamcity;
+import org.apache.ignite.ci.di.AutoProfiling;
+import org.apache.ignite.ci.di.MonitoredTask;
+import org.apache.ignite.ci.di.scheduler.IScheduler;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
+import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class RunHistSync {
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(RunHistSync.class);
+ public static final int MAX_TESTS_QUEUE = 100000;
+ public static final int HIST_LDR_TASKS = 4;
+
+ /** Compactor. */
+ @Inject private IStringCompactor compactor;
+
+ /** Scheduler. */
+ @Inject private IScheduler scheduler;
+
+ /** Run History DAO. */
+ @Inject private RunHistCompactedDao histDao;
+
+ /** Build reference DAO. */
+ @Inject private BuildRefDao buildRefDao;
+
+ /** Build DAO. */
+ @Inject private FatBuildDao fatBuildDao;
+
+ /** Build to save to history. */
+ @GuardedBy("this")
+ private final Map<String, SyncTask> buildToSave = new HashMap<>();
+
+ /**
+ * @param branchName Branch name.
+ */
+ @NotNull public static String normalizeBranch(@Nullable String branchName) {
+ String branch = branchName == null ? ITeamcity.DEFAULT : branchName;
+
+ if (ITeamcity.REFS_HEADS_MASTER.equals(branch))
+ return ITeamcity.DEFAULT;
+
+ if ("master".equals(branch))
+ return ITeamcity.DEFAULT;
+
+ return branch;
+ }
+
+ /**
+ * @param srvVame Server id.
+ * @param build Build.
+ */
+ public void saveToHistoryLater(String srvVame, FatBuildCompacted build) {
+ if (!validForStatistics(build))
+ return;
+
+ int srvId = ITeamcityIgnited.serverIdToInt(srvVame);
+ if (histDao.buildWasProcessed(srvId, build.id()))
+ return;
+
+ boolean saveNow = false;
+
+ int branchNameNormalized = compactor.getStringId(normalizeBranch(build.branchName(compactor)));
+
+ Map<RunHistKey, List<Invocation>> testInvMap = new HashMap<>();
+ build.getAllTests().forEach(t -> {
+ RunHistKey histKey = new RunHistKey(srvId, t.testName(), branchNameNormalized);
+ List<Invocation> list = testInvMap.computeIfAbsent(histKey, k -> new ArrayList<>());
+ Invocation inv = t.toInvocation(compactor, build);
+ list.add(inv);
+ });
+
+ RunHistKey buildInvKey = new RunHistKey(srvId, build.buildTypeId(), branchNameNormalized);
+ Invocation buildInv = build.toInvocation(compactor);
+
+ int cnt = containedTestsCnt(testInvMap);
+
+ synchronized (this) {
+ final SyncTask syncTask = buildToSave.computeIfAbsent(srvVame, s -> new SyncTask());
+
+ if (syncTask.sheduled() + cnt <= MAX_TESTS_QUEUE)
+ syncTask.addLater(testInvMap, buildInvKey, buildInv);
+ else
+ saveNow = true;
+ }
+
+ if (saveNow) {
+ Map<RunHistKey, List<Invocation>> suiteAsMap =
+ Collections.singletonMap(buildInvKey,
+ Collections.singletonList(buildInv));
+ saveInvocationsMap(suiteAsMap, testInvMap);
+ }
+ else {
+ int ldrToActivate = ThreadLocalRandom.current().nextInt(HIST_LDR_TASKS) + 1;
+
+ scheduler.sheduleNamed(taskName("saveBuildToHistory." + ldrToActivate, srvVame),
+ () -> saveBuildToHistory(srvVame, ldrToActivate), 2, TimeUnit.MINUTES);
+ }
+ }
+
+ @MonitoredTask(name="Save Builds To History(srv, runner)", nameExtArgsIndexes = {0,1})
+ @SuppressWarnings("WeakerAccess")
+ protected String saveBuildToHistory(String srvName, int ldrToActivate) {
+ Map<RunHistKey, List<Invocation>> saveThisRun;
+ Map<RunHistKey, List<Invocation>> buildsSaveThisRun;
+
+ synchronized (this) {
+ final SyncTask syncTask = buildToSave.get(srvName);
+ if (syncTask == null)
+ return "Nothing to sync";
+
+ saveThisRun = syncTask.tests;
+ buildsSaveThisRun = syncTask.suites;
+
+ syncTask.tests = new HashMap<>();
+ syncTask.suites = new HashMap<>();
+ }
+
+ return saveInvocationsMap(buildsSaveThisRun, saveThisRun);
+ }
+
+ @AutoProfiling
+ @NotNull protected String saveInvocationsMap(
+ Map<RunHistKey, List<Invocation>> buildsSaveThisRun,
+ Map<RunHistKey, List<Invocation>> saveThisRun) {
+ Set<Integer> confirmedNewBuild = new HashSet<>();
+ Set<Integer> confirmedDuplicate = new HashSet<>();
+
+ AtomicInteger cntTestInvocations = new AtomicInteger();
+ AtomicInteger duplicateOrExpired = new AtomicInteger();
+ AtomicInteger cntSuiteInvocations = new AtomicInteger();
+
+ saveThisRun.forEach(
+ (histKey, invocationList) -> {
+ saveInvocationList(confirmedNewBuild,
+ confirmedDuplicate,
+ cntTestInvocations,
+ duplicateOrExpired,
+ histKey, invocationList);
+ }
+ );
+
+ buildsSaveThisRun.forEach(
+ (histKey, suiteList) -> {
+ List<Invocation> invocationsToSave = suiteList.stream()
+ .filter(inv -> confirmedNewBuild.contains(inv.buildId()))
+ .filter(inv -> !InvocationData.isExpired(inv.startDate()))
+ .collect(Collectors.toList());
+
+ Integer cntAdded = histDao.addSuiteInvocations(histKey, invocationsToSave);
+ cntSuiteInvocations.addAndGet(cntAdded);
+ }
+ );
+
+ String res = "History test entries: " + saveThisRun.size() + " processed " + cntTestInvocations.get()
+ + " invocations saved to DB " + duplicateOrExpired.get() + " duplicates/expired";
+
+ System.out.println(Thread.currentThread().getName() + ":" + res);
+
+ return res;
+ }
+
+ private void saveInvocationList(Set<Integer> confirmedNewBuild,
+ Set<Integer> confirmedDuplicate,
+ AtomicInteger invocations,
+ AtomicInteger duplicateOrExpired,
+ RunHistKey histKey,
+ List<Invocation> invocationList) {
+ List<Invocation> invocationsToSave = new ArrayList<>();
+ invocationList.forEach(
+ inv -> {
+ int buildId = inv.buildId();
+
+ if (confirmedNewBuild.contains(buildId)) {
+ if (!InvocationData.isExpired(inv.startDate()))
+ invocationsToSave.add(inv);
+
+ return;
+ }
+
+ if (confirmedDuplicate.contains(buildId))
+ return;
+
+ if (histDao.setBuildProcessed(histKey.srvId(), buildId, inv.startDate())) {
+ confirmedNewBuild.add(buildId);
+
+ if (!InvocationData.isExpired(inv.startDate()))
+ invocationsToSave.add(inv);
+ }
+ else
+ confirmedDuplicate.add(buildId);
+ }
+ );
+
+ Integer cntAdded = histDao.addTestInvocations(histKey, invocationsToSave);
+
+ invocations.addAndGet(cntAdded);
+ duplicateOrExpired.addAndGet(invocationList.size() - cntAdded);
+ }
+
+ public void invokeLaterFindMissingHistory(String srvName) {
+ scheduler.sheduleNamed(taskName("findMissingHistFromBuildRef", srvName),
+ () -> findMissingHistFromBuildRef(srvName), 360, TimeUnit.MINUTES);
+ }
+
+ @NotNull
+ private String taskName(String taskName, String srvName) {
+ return RunHistSync.class.getSimpleName() +"." + taskName + "." + srvName;
+ }
+
+ @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
+ @MonitoredTask(name = "Find Missing Build History", nameExtArgsIndexes = {0})
+ @AutoProfiling
+ protected String findMissingHistFromBuildRef(String srvId) {
+ int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
+
+ final int[] buildRefKeys = buildRefDao.getAllIds(srvIdMaskHigh);
+
+ List<Integer> buildsIdsToLoad = new ArrayList<>();
+ int totalAskedToLoad = 0;
+
+ for (int buildId : buildRefKeys) {
+ if (histDao.buildWasProcessed(srvIdMaskHigh, buildId))
+ continue;
+
+ if (buildsIdsToLoad.size() >= 50) {
+ totalAskedToLoad += buildsIdsToLoad.size();
+ scheduleHistLoad(srvId, buildsIdsToLoad);
+ buildsIdsToLoad.clear();
+ }
+ buildsIdsToLoad.add(buildId);
+ }
+
+ if (!buildsIdsToLoad.isEmpty()) {
+ totalAskedToLoad += buildsIdsToLoad.size();
+ scheduleHistLoad(srvId, buildsIdsToLoad);
+ }
+
+ return "Invoked later load for history for " + totalAskedToLoad + " builds from " + srvId;
+ }
+
+ /**
+ * @param srvNme Server name;
+ * @param load Build IDs to be loaded into history cache later.
+ */
+ private void scheduleHistLoad(String srvNme, List<Integer> load) {
+ load.forEach(id -> {
+ FatBuildCompacted fatBuild = fatBuildDao.getFatBuild(ITeamcityIgnited.serverIdToInt(srvNme), id);
+
+ if (validForStatistics(fatBuild))
+ saveToHistoryLater(srvNme, fatBuild);
+ else
+ logger.info("Build is not valid for stat: " +
+ (fatBuild != null ? fatBuild.getId() : null));
+ });
+ }
+
+ private boolean validForStatistics(FatBuildCompacted fatBuild) {
+ return fatBuild != null
+ && !fatBuild.isFakeStub()
+ && !fatBuild.isOutdatedEntityVersion()
+ && !fatBuild.isCancelled(compactor)
+ //todo support not finished build reloading usign fat build sync or similar.
+ && fatBuild.isFinished(compactor);
+ }
+
+ /**
+ * Scope of work: builds to be loaded from a connection.
+ */
+ private static class SyncTask {
+ Map<RunHistKey, List<Invocation>> suites = new HashMap<>();
+ Map<RunHistKey, List<Invocation>> tests = new HashMap<>();
+
+ public int sheduled() {
+ return containedTestsCnt(this.tests);
+ }
+
+ public void addLater(Map<RunHistKey, List<Invocation>> testInvMap,
+ RunHistKey buildInvKey,
+ Invocation buildInv) {
+ suites
+ .computeIfAbsent(buildInvKey, k -> new ArrayList<>())
+ .add(buildInv);
+ tests.putAll(testInvMap);
+ }
+ }
+
+ /**
+ * @param tests Tests.
+ * @return count of invocations.
+ */
+ public static int containedTestsCnt(Map<RunHistKey, List<Invocation>> tests) {
+ return tests.values().stream().mapToInt(List::size).sum();
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
index c15a9be..9ab69a7 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
@@ -23,7 +23,7 @@ package org.apache.ignite.ci.web.model;
public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot";
/** TC Bot Version. */
- public static final String VERSION = "20181124";
+ public static final String VERSION = "20181129";
/** TC Bot Version. */
public String version = VERSION;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/ChainAtServerCurrentStatus.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/ChainAtServerCurrentStatus.java
index 0c023b0..975d8f6 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/ChainAtServerCurrentStatus.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/ChainAtServerCurrentStatus.java
@@ -22,14 +22,18 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import org.apache.ignite.ci.ITcAnalytics;
import org.apache.ignite.ci.ITeamcity;
import org.apache.ignite.ci.analysis.FullChainRunCtx;
-import org.apache.ignite.ci.analysis.ITestFailures;
+import org.apache.ignite.ci.analysis.IMultTestOccurrence;
import org.apache.ignite.ci.analysis.MultBuildRunCtx;
import org.apache.ignite.ci.tcmodel.conf.BuildType;
+import org.apache.ignite.ci.analysis.TestInBranch;
+import org.apache.ignite.ci.teamcity.ignited.IRunHistory;
+import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.util.CollectionUtil;
import org.apache.ignite.internal.util.typedef.T2;
@@ -92,10 +96,11 @@ public class ChainAtServerCurrentStatus {
this.branchName = branchTc;
}
- public void initFromContext(ITeamcity teamcity,
- FullChainRunCtx ctx,
- ITcAnalytics tcAnalytics,
- @Nullable String baseBranchTc) {
+ public void initFromContext(ITeamcityIgnited tcIgnited,
+ @Deprecated ITeamcity teamcity,
+ FullChainRunCtx ctx,
+ @Deprecated ITcAnalytics tcAnalytics,
+ @Nullable String baseBranchTc) {
failedTests = 0;
failedToFinish = 0;
//todo mode with not failed
@@ -105,7 +110,7 @@ public class ChainAtServerCurrentStatus {
suite -> {
final SuiteCurrentStatus suiteCurStatus = new SuiteCurrentStatus();
- suiteCurStatus.initFromContext(teamcity, suite, tcAnalytics, baseBranchTc);
+ suiteCurStatus.initFromContext(tcIgnited, teamcity, suite, tcAnalytics, baseBranchTc);
failedTests += suiteCurStatus.failedTests;
if (suite.hasAnyBuildProblemExceptTestOrSnapshot() || suite.onlyCancelledBuilds())
@@ -120,18 +125,24 @@ public class ChainAtServerCurrentStatus {
webToHist = buildWebLink(teamcity, ctx);
webToBuild = buildWebLinkToBuild(teamcity, ctx);
- Stream<T2<MultBuildRunCtx, ITestFailures>> allLongRunning = ctx.suites().flatMap(
+ Stream<T2<MultBuildRunCtx, IMultTestOccurrence>> allLongRunning = ctx.suites().flatMap(
suite -> suite.getTopLongRunning().map(t -> new T2<>(suite, t))
);
- Comparator<T2<MultBuildRunCtx, ITestFailures>> durationComp
+ Comparator<T2<MultBuildRunCtx, IMultTestOccurrence>> durationComp
= Comparator.comparing((pair) -> pair.get2().getAvgDurationMs());
CollectionUtil.top(allLongRunning, 3, durationComp).forEach(
pairCtxAndOccur -> {
MultBuildRunCtx suite = pairCtxAndOccur.get1();
- ITestFailures longRunningOccur = pairCtxAndOccur.get2();
+ IMultTestOccurrence longRunningOccur = pairCtxAndOccur.get2();
- TestFailure failure = createOrrucForLongRun(teamcity, suite, tcAnalytics, longRunningOccur, baseBranchTc);
+ Function<TestInBranch, ? extends IRunHistory> function = SuiteCurrentStatus.NEW_RUN_STAT
+ ? tcIgnited::getTestRunHist
+ : tcAnalytics.getTestRunStatProvider();
+
+ TestFailure failure = createOrrucForLongRun(tcIgnited, suite, tcAnalytics, longRunningOccur,
+ baseBranchTc,
+ function);
failure.testName = "[" + suite.suiteName() + "] " + failure.testName; //may be separate field
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
index 2216edc..f595b71 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
@@ -26,12 +26,11 @@ import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.ignite.ci.ITcAnalytics;
import org.apache.ignite.ci.ITeamcity;
-import org.apache.ignite.ci.analysis.ITestFailures;
+import org.apache.ignite.ci.analysis.IMultTestOccurrence;
import org.apache.ignite.ci.analysis.MultBuildRunCtx;
import org.apache.ignite.ci.analysis.RunStat;
import org.apache.ignite.ci.analysis.SuiteInBranch;
@@ -39,11 +38,13 @@ import org.apache.ignite.ci.analysis.TestInBranch;
import org.apache.ignite.ci.analysis.TestLogCheckResult;
import org.apache.ignite.ci.issue.EventTemplates;
import org.apache.ignite.ci.issue.ProblemRef;
+import org.apache.ignite.ci.teamcity.ignited.IRunHistory;
+import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.web.model.hist.FailureSummary;
import org.apache.ignite.ci.web.rest.GetBuildLog;
import org.jetbrains.annotations.NotNull;
-import static org.apache.ignite.ci.tcbot.chain.BuildChainProcessor.normalizeBranch;
+import static org.apache.ignite.ci.teamcity.ignited.runhist.RunHistSync.normalizeBranch;
import static org.apache.ignite.ci.util.TimeUtil.millisToDurationPrintable;
import static org.apache.ignite.ci.util.UrlUtil.escape;
@@ -51,6 +52,9 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
* Represent Suite result
*/
@SuppressWarnings("WeakerAccess") public class SuiteCurrentStatus extends FailureSummary {
+ /** Use New run stat in PR analysis. */
+ public static final boolean NEW_RUN_STAT = false;
+
/** Suite Name */
public String name;
@@ -121,18 +125,29 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
/** Possible blocker: filled for PR and builds checks, mean there was stable execution in master, but */
public Boolean possibleBlocker;
- public void initFromContext(@Nonnull final ITeamcity teamcity,
- @Nonnull final MultBuildRunCtx suite,
- @NotNull final ITcAnalytics tcAnalytics,
- @Nullable final String baseBranch) {
+ public void initFromContext(ITeamcityIgnited tcIgnited,
+ @Nonnull final ITeamcity teamcity,
+ @Nonnull final MultBuildRunCtx suite,
+ @NotNull final ITcAnalytics tcAnalytics,
+ @Nullable final String baseBranch) {
name = suite.suiteName();
String failRateNormalizedBranch = normalizeBranch(baseBranch);
String curBranchNormalized = normalizeBranch(suite.branchName());
+ Function<TestInBranch, ? extends IRunHistory> testStatProv = NEW_RUN_STAT
+ ? tcIgnited::getTestRunHist
+ : tcAnalytics.getTestRunStatProvider();
+
String suiteId = suite.suiteId();
- initStat(tcAnalytics, failRateNormalizedBranch, curBranchNormalized, suiteId);
+
+ Function<SuiteInBranch, ? extends IRunHistory> provider =
+ NEW_RUN_STAT
+ ? tcIgnited::getSuiteRunHist
+ : tcAnalytics.getBuildFailureRunStatProvider();
+
+ initSuiteStat(provider, failRateNormalizedBranch, curBranchNormalized, suiteId);
Set<String> collect = suite.lastChangeUsers().collect(Collectors.toSet());
@@ -148,11 +163,11 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
webToHistBaseBranch = buildWebLink(teamcity, suite, baseBranch);
webToBuild = buildWebLinkToBuild(teamcity, suite);
- List<ITestFailures> tests = suite.getFailedTests();
- Function<ITestFailures, Float> function = foccur -> {
+ List<IMultTestOccurrence> tests = suite.getFailedTests();
+ Function<IMultTestOccurrence, Float> function = foccur -> {
TestInBranch testInBranch = new TestInBranch(foccur.getName(), failRateNormalizedBranch);
- RunStat apply = tcAnalytics.getTestRunStatProvider().apply(testInBranch);
+ IRunHistory apply = testStatProv.apply(testInBranch);
return apply == null ? 0f : apply.getFailRate();
};
@@ -161,14 +176,15 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
tests.forEach(occurrence -> {
final TestFailure failure = new TestFailure();
- failure.initFromOccurrence(occurrence, teamcity, suite.projectId(), suite.branchName(), baseBranch);
- failure.initStat(tcAnalytics.getTestRunStatProvider(), failRateNormalizedBranch, curBranchNormalized);
+ failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), baseBranch);
+ failure.initStat(testStatProv, failRateNormalizedBranch, curBranchNormalized);
testFailures.add(failure);
});
suite.getTopLongRunning().forEach(occurrence -> {
- final TestFailure failure = createOrrucForLongRun(teamcity, suite, tcAnalytics, occurrence, baseBranch);
+ final TestFailure failure = createOrrucForLongRun(tcIgnited, suite, tcAnalytics,
+ occurrence, baseBranch, testStatProv);
topLongRunning.add(failure);
});
@@ -211,13 +227,16 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
// todo implement this logic in suite possibleBlocker = suite.hasPossibleBlocker();
}
- private void initStat(@Nullable ITcAnalytics tcAnalytics, String failRateNormalizedBranch, String curBranchNormalized, String suiteId) {
- if (Strings.isNullOrEmpty(suiteId) || tcAnalytics == null)
+ private void initSuiteStat(Function<SuiteInBranch, ? extends IRunHistory> suiteFailProv,
+ String failRateNormalizedBranch,
+ String curBranchNormalized,
+ String suiteId) {
+ if (Strings.isNullOrEmpty(suiteId) )
return;
SuiteInBranch key = new SuiteInBranch(suiteId, failRateNormalizedBranch);
- final RunStat stat = tcAnalytics.getBuildFailureRunStatProvider().apply(key);
+ final IRunHistory stat = suiteFailProv.apply(key);
if (stat != null) {
failures = stat.getFailuresCount();
@@ -235,24 +254,25 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
latestRuns = stat.getLatestRunResults();
}
- RunStat latestRunsSrc = null;
+ IRunHistory latestRunsSrc = null;
if (!failRateNormalizedBranch.equals(curBranchNormalized)) {
SuiteInBranch keyForStripe = new SuiteInBranch(suiteId, curBranchNormalized);
- final RunStat statForStripe = tcAnalytics.getBuildFailureRunStatProvider().apply(keyForStripe);
+ final IRunHistory statForStripe = suiteFailProv.apply(keyForStripe);
latestRunsSrc = statForStripe;
latestRuns = statForStripe != null ? statForStripe.getLatestRunResults() : null;
} else
latestRunsSrc = stat;
- if (latestRunsSrc != null) {
- RunStat.TestId testId = latestRunsSrc.detectTemplate(EventTemplates.newFailureForFlakyTest); //extended runs required for suite
+ if (latestRunsSrc instanceof RunStat) {
+ RunStat latestRunsSrcV1 = (RunStat)latestRunsSrc;
+ RunStat.TestId testId = latestRunsSrcV1.detectTemplate(EventTemplates.newFailureForFlakyTest); //extended runs required for suite
if (testId != null)
problemRef = new ProblemRef("New Failure");
- RunStat.TestId buildIdCritical = latestRunsSrc.detectTemplate(EventTemplates.newCriticalFailure);
+ RunStat.TestId buildIdCritical = latestRunsSrcV1.detectTemplate(EventTemplates.newCriticalFailure);
if (buildIdCritical != null)
problemRef = new ProblemRef("New Critical Failure");
@@ -267,17 +287,18 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
return failure;
}
- @NotNull public static TestFailure createOrrucForLongRun(@Nonnull ITeamcity teamcity,
+ @NotNull public static TestFailure createOrrucForLongRun(ITeamcityIgnited tcIgnited,
@Nonnull MultBuildRunCtx suite,
@Nullable final ITcAnalytics tcAnalytics,
- final ITestFailures occurrence,
- @Nullable final String failRateBranch) {
+ final IMultTestOccurrence occurrence,
+ @Nullable final String failRateBranch,
+ Function<TestInBranch, ? extends IRunHistory> supplier) {
final TestFailure failure = new TestFailure();
- failure.initFromOccurrence(occurrence, teamcity, suite.projectId(), suite.branchName(), failRateBranch);
+ failure.initFromOccurrence(occurrence, tcIgnited, suite.projectId(), suite.branchName(), failRateBranch);
if (tcAnalytics != null) {
- failure.initStat(tcAnalytics.getTestRunStatProvider(),
+ failure.initStat(supplier,
normalizeBranch(failRateBranch),
normalizeBranch(suite.branchName()));
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java
index c3c28a3..9748911 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java
@@ -26,13 +26,14 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-import org.apache.ignite.ci.ITeamcity;
-import org.apache.ignite.ci.analysis.ITestFailures;
+import org.apache.ignite.ci.analysis.IMultTestOccurrence;
import org.apache.ignite.ci.analysis.RunStat;
import org.apache.ignite.ci.analysis.TestInBranch;
import org.apache.ignite.ci.issue.EventTemplates;
import org.apache.ignite.ci.issue.ProblemRef;
import org.apache.ignite.ci.logs.LogMsgToWarn;
+import org.apache.ignite.ci.teamcity.ignited.IRunHistory;
+import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.web.model.hist.FailureSummary;
import org.apache.ignite.ci.web.model.hist.TestHistory;
import org.jetbrains.annotations.NotNull;
@@ -92,13 +93,13 @@ import static org.apache.ignite.ci.web.model.current.SuiteCurrentStatus.branchFo
/**
* @param failure
- * @param teamcity
+ * @param tcIgn
* @param projectId
* @param branchName
* @param baseBranchName base branch name (e.g. master).
*/
- public void initFromOccurrence(@Nonnull final ITestFailures failure,
- @Nonnull final ITeamcity teamcity,
+ public void initFromOccurrence(@Nonnull final IMultTestOccurrence failure,
+ @Nonnull final ITeamcityIgnited tcIgn,
@Nullable final String projectId,
@Nullable final String branchName,
@Nullable final String baseBranchName) {
@@ -137,11 +138,11 @@ import static org.apache.ignite.ci.web.model.current.SuiteCurrentStatus.branchFo
}
if (webUrl == null)
if (full.test != null && full.test.id != null)
- webUrl = buildWebLink(teamcity, full.test.id, projectId, branchName);
+ webUrl = buildWebLink(tcIgn, full.test.id, projectId, branchName);
if (webUrlBaseBranch == null)
if (full.test != null && full.test.id != null)
- webUrlBaseBranch = buildWebLink(teamcity, full.test.id, projectId, baseBranchName);
+ webUrlBaseBranch = buildWebLink(tcIgn, full.test.id, projectId, baseBranchName);
});
}
@@ -173,12 +174,12 @@ import static org.apache.ignite.ci.web.model.current.SuiteCurrentStatus.branchFo
}
}
- private static String buildWebLink(ITeamcity teamcity, Long id,
+ private static String buildWebLink(ITeamcityIgnited tcIgn, Long id,
@Nullable String projectId, @Nullable String branchName) {
if (projectId == null)
return null;
final String branch = branchForLink(branchName);
- return teamcity.host() + "project.html"
+ return tcIgn.host() + "project.html"
+ "?projectId=" + projectId
+ "&testNameId=" + id
+ "&branch=" + escape(branch)
@@ -190,7 +191,7 @@ import static org.apache.ignite.ci.web.model.current.SuiteCurrentStatus.branchFo
* @param failRateNormalizedBranch Base branch: Fail rate and flakyness detection normalized branch.
* @param curBranchNormalized Cur branch normalized.
*/
- public void initStat(@Nullable final Function<TestInBranch, RunStat> runStatSupplier,
+ public void initStat(@Nullable final Function<TestInBranch, ? extends IRunHistory> runStatSupplier,
String failRateNormalizedBranch,
String curBranchNormalized) {
if (runStatSupplier == null)
@@ -198,11 +199,11 @@ import static org.apache.ignite.ci.web.model.current.SuiteCurrentStatus.branchFo
TestInBranch testInBranch = new TestInBranch(name, failRateNormalizedBranch);
- final RunStat stat = runStatSupplier.apply(testInBranch);
+ final IRunHistory stat = runStatSupplier.apply(testInBranch);
histBaseBranch.init(stat);
- RunStat statForProblemsDetection = null;
+ IRunHistory statForProblemsDetection = null;
if (!curBranchNormalized.equals(failRateNormalizedBranch)) {
TestInBranch testInBranchS = new TestInBranch(name, curBranchNormalized);
@@ -217,13 +218,14 @@ import static org.apache.ignite.ci.web.model.current.SuiteCurrentStatus.branchFo
} else
statForProblemsDetection = stat;
- if (statForProblemsDetection != null) {
- RunStat.TestId testId = statForProblemsDetection.detectTemplate(EventTemplates.newFailure);
+ if (statForProblemsDetection instanceof RunStat) {
+ final RunStat forProblemsDetection = (RunStat) statForProblemsDetection;
+ RunStat.TestId testId = forProblemsDetection.detectTemplate(EventTemplates.newFailure);
if (testId != null)
problemRef = new ProblemRef("New Failure");
- RunStat.TestId recentContributedTestId = statForProblemsDetection.detectTemplate(EventTemplates.newContributedTestFailure);
+ RunStat.TestId recentContributedTestId = forProblemsDetection.detectTemplate(EventTemplates.newContributedTestFailure);
if (recentContributedTestId != null)
problemRef = new ProblemRef("Recently contributed test failure");
@@ -246,6 +248,8 @@ import static org.apache.ignite.ci.web.model.current.SuiteCurrentStatus.branchFo
boolean lowFailureRate = recent != null && recent.failureRate != null &&
Float.valueOf(recent.failureRate.replace(',', '.')) < 4.;
+ //System.out.println(name + " " + recent.failureRate);
+
return lowFailureRate && histBaseBranch.flakyComments == null;
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/TestHistory.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/TestHistory.java
index d25ec92..f18fe20 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/TestHistory.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/TestHistory.java
@@ -20,18 +20,17 @@ package org.apache.ignite.ci.web.model.hist;
import com.google.common.base.Objects;
import java.util.List;
import javax.annotation.Nullable;
-import org.apache.ignite.ci.analysis.RunStat;
-import org.jetbrains.annotations.NotNull;
+import org.apache.ignite.ci.teamcity.ignited.IRunHistory;
/**
* Summary of failures - all history and recent runs for suite or for suite.
*/
public class TestHistory {
/** 'All the time' runs history statistic. */
- @Nullable public FailureSummary allTime = new FailureSummary();
+ public FailureSummary allTime = new FailureSummary();
/** Latest runs history statistic. */
- @Nullable public FailureSummary recent = new FailureSummary();
+ public FailureSummary recent = new FailureSummary();
/** Latest runs, 0,1,2 values for each run. */
@Nullable public List<Integer> latestRuns;
@@ -39,7 +38,7 @@ public class TestHistory {
/** Non null flaky comments means there is flakiness detected in the the branch. */
@Nullable public String flakyComments;
- public void init(@Nullable RunStat stat) {
+ public void init(@Nullable IRunHistory stat) {
if (stat == null)
return;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/GetChainResultsAsHtml.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/GetChainResultsAsHtml.java
index a84afc2..1b72d4c 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/GetChainResultsAsHtml.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/GetChainResultsAsHtml.java
@@ -75,10 +75,10 @@ public class GetChainResultsAsHtml {
ITcServerProvider tcHelper = injector.getInstance(ITcServerProvider.class);
final ICredentialsProv creds = ICredentialsProv.get(req);
IAnalyticsEnabledTeamcity teamcity = tcHelper.server(srvId, creds);
- ITeamcityIgnited teamcityIgnited = injector.getInstance(ITeamcityIgnitedProvider.class).server(srvId, creds);
+ ITeamcityIgnited tcIgn = injector.getInstance(ITeamcityIgnitedProvider.class).server(srvId, creds);
final FullChainRunCtx ctx = buildChainProcessor.loadFullChainContext(teamcity,
- teamcityIgnited,
+ tcIgn,
Collections.singletonList(buildId),
LatestRebuildMode.NONE,
ProcessLogsMode.SUITE_NOT_COMPLETE,
@@ -92,7 +92,7 @@ public class GetChainResultsAsHtml {
status.chainName = ctx.suiteName();
- status.initFromContext(teamcity, ctx, teamcity, failRateBranch);
+ status.initFromContext(tcIgn, teamcity, ctx, teamcity, failRateBranch);
res.append(showChainAtServerData(status));
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java
index f0df8f8..0ad6c3e 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java
@@ -19,22 +19,19 @@ package org.apache.ignite.ci.web.rest.build;
import com.google.common.collect.BiMap;
import java.text.ParseException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
+
import com.google.inject.Injector;
import org.apache.ignite.ci.tcbot.trends.MasterTrendsService;
import org.apache.ignite.ci.teamcity.ignited.SyncMode;
import org.apache.ignite.ci.teamcity.ignited.buildcondition.BuildCondition;
import org.apache.ignite.ci.tcbot.chain.BuildChainProcessor;
import org.apache.ignite.ci.IAnalyticsEnabledTeamcity;
-import org.apache.ignite.ci.ITcHelper;
import org.apache.ignite.ci.ITeamcity;
import org.apache.ignite.ci.analysis.FullChainRunCtx;
import org.apache.ignite.ci.analysis.mode.LatestRebuildMode;
import org.apache.ignite.ci.analysis.mode.ProcessLogsMode;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnitedProvider;
-import org.apache.ignite.ci.tcmodel.result.tests.TestRef;
import org.apache.ignite.ci.teamcity.restcached.ITcServerProvider;
import org.apache.ignite.ci.user.ICredentialsProv;
import org.apache.ignite.ci.web.model.current.BuildStatisticsSummary;
@@ -132,14 +129,14 @@ public class GetBuildTestFailures {
throw ServiceUnauthorizedException.noCreds(srvId);
IAnalyticsEnabledTeamcity teamcity = tcSrvProvider.server(srvId, prov);
- ITeamcityIgnited teamcityIgnited = tcIgnitedProv.server(srvId, prov);
+ ITeamcityIgnited tcIgnited = tcIgnitedProv.server(srvId, prov);
String failRateBranch = ITeamcity.DEFAULT;
ProcessLogsMode procLogs = (checkAllLogs != null && checkAllLogs) ? ProcessLogsMode.ALL : ProcessLogsMode.SUITE_NOT_COMPLETE;
final FullChainRunCtx ctx = buildChainProcessor.loadFullChainContext(teamcity,
- teamcityIgnited,
+ tcIgnited,
Collections.singletonList(buildId),
LatestRebuildMode.NONE,
procLogs,
@@ -153,7 +150,7 @@ public class GetBuildTestFailures {
if (cnt > 0)
runningUpdates.addAndGet(cnt);
- chainStatus.initFromContext(teamcity, ctx, teamcity, failRateBranch);
+ chainStatus.initFromContext(tcIgnited, teamcity, ctx, teamcity, failRateBranch);
res.addChainOnServer(chainStatus);
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java
index c2477dc..8e6390c 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java
@@ -25,7 +25,7 @@ import com.google.inject.internal.SingletonScope;
import org.apache.ignite.ci.IAnalyticsEnabledTeamcity;
import org.apache.ignite.ci.ITeamcity;
import org.apache.ignite.ci.analysis.FullChainRunCtx;
-import org.apache.ignite.ci.analysis.ITestFailures;
+import org.apache.ignite.ci.analysis.IMultTestOccurrence;
import org.apache.ignite.ci.analysis.MultBuildRunCtx;
import org.apache.ignite.ci.analysis.mode.LatestRebuildMode;
import org.apache.ignite.ci.analysis.mode.ProcessLogsMode;
@@ -99,7 +99,7 @@ public class BuildChainProcessorTest {
else
assertTrue(suite.failedTests() >= 1);
- for (ITestFailures test : suite.getFailedTests()) {
+ for (IMultTestOccurrence test : suite.getFailedTests()) {
if (test.getName().startsWith(UNIQUE_FAILED_TEST))
assertEquals(1, test.failuresCount());
else if (test.getName().equals(TEST_FAILING_EVERY_TIME))
@@ -131,7 +131,7 @@ public class BuildChainProcessorTest {
System.out.println(suite.getFailedTestsNames().collect(Collectors.toList()));
if (suite.suiteName() != null && suite.suiteName().startsWith(UNIQUE_FAILED_TEST)) {
- for (ITestFailures test : suite.getFailedTests())
+ for (IMultTestOccurrence test : suite.getFailedTests())
fail("Failure found but should be hidden by re-run " + test.getName());
}
}
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java
index 7d57d28..48201b3 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessorTest.java
@@ -16,11 +16,23 @@
*/
package org.apache.ignite.ci.tcbot.chain;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.internal.SingletonScope;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
import org.apache.ignite.ci.IAnalyticsEnabledTeamcity;
+import org.apache.ignite.ci.ITeamcity;
+import org.apache.ignite.ci.analysis.RunStat;
import org.apache.ignite.ci.github.pure.IGitHubConnection;
import org.apache.ignite.ci.github.pure.IGitHubConnectionProvider;
import org.apache.ignite.ci.tcmodel.conf.BuildType;
@@ -30,21 +42,22 @@ import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrence;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.tcmodel.result.tests.TestRef;
-import org.apache.ignite.ci.teamcity.ignited.*;
+import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnitedProvider;
+import org.apache.ignite.ci.teamcity.ignited.InMemoryStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.TeamcityIgnitedProviderMock;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
import org.apache.ignite.ci.teamcity.restcached.ITcServerProvider;
import org.apache.ignite.ci.user.ICredentialsProv;
import org.apache.ignite.ci.web.model.current.SuiteCurrentStatus;
+import org.apache.ignite.ci.web.model.current.TestFailure;
import org.jetbrains.annotations.NotNull;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -55,6 +68,14 @@ import static org.mockito.Mockito.when;
public class PrChainsProcessorTest {
public static final String SRV_ID = "apache";
+ public static final String TEST_WITH_HISTORY_FAILING_IN_MASTER = "testWithHistoryFailingInMaster";
+ public static final String TEST_WITH_HISTORY_PASSING_IN_MASTER = "testWithHistoryPassingInMaster";
+ public static final String TEST_FLAKY_IN_MASTER = "testFlaky50";
+ public static final String TEST_WITHOUT_HISTORY = "testWithoutHistory";
+ public static final String TEST_WAS_FIXED_IN_MASTER = "testFailingButFixedInMaster";
+ public static final int NUM_OF_TESTS_IN_MASTER = 10;
+ public static final String CACHE_1 = "Cache1";
+
private Map<Integer, FatBuildCompacted> apacheBuilds = new ConcurrentHashMap<>();
@@ -77,7 +98,6 @@ public class PrChainsProcessorTest {
final IAnalyticsEnabledTeamcity tcOld = BuildChainProcessorTest.tcOldMock();
when(tcSrvOldProv.server(anyString(), any(ICredentialsProv.class))).thenReturn(tcOld);
-
bind(ITcServerProvider.class).toInstance(tcSrvOldProv);
super.configure();
@@ -100,66 +120,176 @@ public class PrChainsProcessorTest {
initBuildChain(c, btId, branch);
PrChainsProcessor prcp = injector.getInstance(PrChainsProcessor.class);
- final List<SuiteCurrentStatus> suitesStatuses = prcp.getSuitesStatuses(btId,
+ final List<SuiteCurrentStatus> blockers = prcp.getBlockersSuitesStatuses(btId,
branch, SRV_ID, mock(ICredentialsProv.class));
- assertNotNull(suitesStatuses);
- assertFalse(suitesStatuses.isEmpty());
+ System.out.println(blockers);
+ assertNotNull(blockers);
+ assertFalse(blockers.isEmpty());
- assertTrue(suitesStatuses.stream().anyMatch(
- s -> s.testFailures.stream().anyMatch(testFailure -> "testWithoutHistory".equals(testFailure.name))
- ));
+ assertTrue(blockers.stream().anyMatch(containsTestFail("testWithoutHistory")));
- assertTrue(suitesStatuses.stream().anyMatch(s -> "Build".equals(s.suiteId)));
- assertTrue(suitesStatuses.stream().anyMatch(s -> "CancelledBuild".equals(s.suiteId)));
- }
+ assertTrue(blockers.stream().anyMatch(s -> "Build".equals(s.suiteId)));
+ assertTrue(blockers.stream().anyMatch(s -> "CancelledBuild".equals(s.suiteId)));
- private void initBuildChain(IStringCompactor c, String btId, String branch) {
- final int id = 1000;
+ assertTrue(blockers.stream().anyMatch(containsTestFail(TEST_WITH_HISTORY_PASSING_IN_MASTER)));
- final FatBuildCompacted chain = createFailedBuild(c, btId, branch, id, 100000);
+ if (SuiteCurrentStatus.NEW_RUN_STAT) {
+ assertFalse(blockers.stream().anyMatch(containsTestFail(TEST_WITH_HISTORY_FAILING_IN_MASTER)));
+ assertFalse(blockers.stream().anyMatch(containsTestFail(TEST_FLAKY_IN_MASTER)));
+ }
- final FatBuildCompacted childBuild = createFailedBuild(c, "Cache1", branch, 1001, 100020);
- TestOccurrenceFull tf = new TestOccurrenceFull();
- tf.test = new TestRef();
- tf.test.id = 1L;
- tf.name = "testWithoutHistory";
- tf.status = TestOccurrence.STATUS_FAILURE;
+ Optional<SuiteCurrentStatus> suiteOpt = blockers.stream().filter(containsTestFail(TEST_WITH_HISTORY_PASSING_IN_MASTER)).findAny();
+ assertTrue(suiteOpt.isPresent());
+ Optional<TestFailure> testOpt = suiteOpt.get().testFailures.stream().filter(tf -> TEST_WITH_HISTORY_PASSING_IN_MASTER.equals(tf.name)).findAny();
+ assertTrue(testOpt.isPresent());
- childBuild.addTests(c, Collections.singletonList(tf));
+ if (SuiteCurrentStatus.NEW_RUN_STAT) {
+ List<Integer> etalon = new ArrayList<>();
+ for (int i = 0; i < NUM_OF_TESTS_IN_MASTER; i++)
+ etalon.add(RunStat.RunStatus.RES_OK.getCode());
+
+ assertEquals(etalon, testOpt.get().histBaseBranch.latestRuns);
+ }
+
+ assertTrue(blockers.stream().anyMatch(containsTestFail(TEST_WAS_FIXED_IN_MASTER)));
+ if(SuiteCurrentStatus.NEW_RUN_STAT)
+ assertFalse(blockers.stream().anyMatch(containsTestFail(TEST_WITH_HISTORY_FAILING_IN_MASTER)));
+ // otherwise this non-blocker will not be filtered out
+
+ assertTrue(blockers.stream().anyMatch(containsTestFail(TEST_WITH_HISTORY_PASSING_IN_MASTER)));
+ }
+ @NotNull
+ private Predicate<SuiteCurrentStatus> containsTestFail(String name) {
+ return s -> s.testFailures.stream().anyMatch(testFailure -> {
+ return name.equals(testFailure.name);
+ });
+ }
+ public void initBuildChain(IStringCompactor c, String btId, String branch) {
final FatBuildCompacted buildBuild = createFailedBuild(c, "Build", branch, 1002, 100020);
final ProblemOccurrence compile = new ProblemOccurrence();
compile.setType(ProblemOccurrence.TC_COMPILATION_ERROR);
buildBuild.addProblems(c, Collections.singletonList(compile));
- childBuild.snapshotDependencies(new int[]{buildBuild.id()});
- final Build build = createBuild("CancelledBuild", branch, 1003, 100020);
+ final FatBuildCompacted childBuild =
+ createFailedBuild(c, CACHE_1, branch, 1001, 100020)
+ .addTests(c,
+ Lists.newArrayList(
+ createFailedTest(1L, TEST_WITHOUT_HISTORY),
+ createFailedTest(2L, TEST_WITH_HISTORY_FAILING_IN_MASTER),
+ createFailedTest(3L, TEST_WITH_HISTORY_PASSING_IN_MASTER),
+ createFailedTest(50L, TEST_FLAKY_IN_MASTER),
+ createFailedTest(400L, TEST_WAS_FIXED_IN_MASTER)));
+
+ childBuild.snapshotDependencies(new int[] {buildBuild.id()});
+
+ final Build build = createJaxbBuild("CancelledBuild", branch, 1003, 100020, true);
+
build.status = BuildRef.STATUS_UNKNOWN;
build.state = BuildRef.STATE_FINISHED;
final FatBuildCompacted cancelledBuild = new FatBuildCompacted(c, build);
+ cancelledBuild.snapshotDependencies(new int[]{buildBuild.id()});
+
+ final int id = 1000;
+
+ final FatBuildCompacted chain = createFailedBuild(c, btId, branch, id, 100000);
chain.snapshotDependencies(new int[]{childBuild.id(), cancelledBuild.id()});
- apacheBuilds.put(chain.id(), chain);
- apacheBuilds.put(childBuild.id(), childBuild);
- apacheBuilds.put(buildBuild.id(), buildBuild);
- apacheBuilds.put(cancelledBuild.id(), cancelledBuild);
+ addBuilds(chain);
+ addBuilds(childBuild);
+ addBuilds(buildBuild);
+ addBuilds(cancelledBuild);
+
+ for (int i = 0; i < NUM_OF_TESTS_IN_MASTER; i++) {
+ FatBuildCompacted cache1InMaster = createFailedBuild(c, CACHE_1,
+ ITeamcity.DEFAULT, 500 + i, 100000 + (i * 10000))
+ .addTests(c, Lists.newArrayList(
+ createFailedTest(2L, TEST_WITH_HISTORY_FAILING_IN_MASTER),
+ createPassingTest(3L, TEST_WITH_HISTORY_PASSING_IN_MASTER),
+ createTest(50L, TEST_FLAKY_IN_MASTER, i % 2 == 0),
+ createPassingTest(400L, TEST_WAS_FIXED_IN_MASTER)));
+
+ if (i % 7 == 1) {
+ ProblemOccurrence timeout = new ProblemOccurrence();
+ timeout.setType(ProblemOccurrence.TC_EXECUTION_TIMEOUT);
+ cache1InMaster.addProblems(c, Collections.singletonList(timeout));
+ }
+
+ addBuilds(cache1InMaster);
+ }
+
+ long ageMs = TimeUnit.DAYS.toMillis(InvocationData.MAX_DAYS);
+
+ for (int i = 0; i < 134; i++) {
+ addBuilds(createFailedBuild(c, CACHE_1,
+ ITeamcity.DEFAULT, i, ageMs + (i * 10000))
+ .addTests(c, Lists.newArrayList(
+ createFailedTest(400L, TEST_WAS_FIXED_IN_MASTER))));
+ }
+
+ for (int i = 0; i < 10; i++) {
+ final FatBuildCompacted successfull =
+ createFatBuild(c, CACHE_1, "some-exotic-branch", i+7777, 100020, true)
+ .addTests(c,
+ Lists.newArrayList(
+ createPassingTest(1L, TEST_WITHOUT_HISTORY),
+ createPassingTest(2L, TEST_WITH_HISTORY_FAILING_IN_MASTER),
+ createPassingTest(3L, TEST_WITH_HISTORY_PASSING_IN_MASTER),
+ createPassingTest(50L, TEST_FLAKY_IN_MASTER),
+ createPassingTest(400L, TEST_WAS_FIXED_IN_MASTER)));
+
+ addBuilds(successfull);
+ }
+ }
+
+ private void addBuilds(FatBuildCompacted... builds) {
+ for (FatBuildCompacted build : builds) {
+ final FatBuildCompacted oldB = apacheBuilds.put(build.id(), build);
+
+ Preconditions.checkState(oldB==null);
+ }
}
@NotNull
- public FatBuildCompacted createFailedBuild(IStringCompactor c, String btId, String branch, int id, int ageMs) {
- final Build build = createBuild(btId, branch, id, ageMs);
- build.status = BuildRef.STATUS_FAILURE;
+ private TestOccurrenceFull createFailedTest(long id, String name) {
+ return createTest(id, name, false);
+ }
+
+ @NotNull
+ private TestOccurrenceFull createPassingTest(long id, String name) {
+ return createTest(id, name, true);
+ }
+
+ @NotNull private TestOccurrenceFull createTest(long id, String name, boolean passed) {
+ TestOccurrenceFull tf = new TestOccurrenceFull();
+
+ tf.test = new TestRef();
+
+ tf.test.id = id;
+ tf.name = name;
+ tf.status = passed ? TestOccurrence.STATUS_SUCCESS : TestOccurrence.STATUS_FAILURE;
+ return tf;
+ }
+
+ @NotNull
+ public FatBuildCompacted createFailedBuild(IStringCompactor c, String btId, String branch, int id, long ageMs) {
+ return createFatBuild(c, btId, branch, id, ageMs, false);
+ }
+
+ @NotNull public FatBuildCompacted createFatBuild(IStringCompactor c, String btId, String branch, int id, long ageMs,
+ boolean passed) {
+ final Build build = createJaxbBuild(btId, branch, id, ageMs, passed);
return new FatBuildCompacted(c, build);
}
@NotNull
- public Build createBuild(String btId, String branch, int id, int ageMs) {
+ private Build createJaxbBuild(String btId, String branch, int id, long ageMs, boolean passed) {
final Build build = new Build();
build.buildTypeId = btId;
final BuildType type = new BuildType();
@@ -169,6 +299,13 @@ public class PrChainsProcessorTest {
build.setId(id);
build.setStartDateTs(System.currentTimeMillis() - ageMs);
build.setBranchName(branch);
+ build.state = Build.STATE_FINISHED;
+ build.status = passed ? Build.STATUS_SUCCESS : BuildRef.STATUS_FAILURE;
+
return build;
}
+
+ public Map<Integer, FatBuildCompacted> apacheBuilds() {
+ return apacheBuilds;
+ }
}
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index 6aa39bc..c472447 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -25,20 +25,22 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
+import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import javax.xml.bind.JAXBException;
+
+import com.google.inject.internal.SingletonScope;
import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
+import org.apache.ignite.ci.ITeamcity;
+import org.apache.ignite.ci.analysis.SuiteInBranch;
+import org.apache.ignite.ci.analysis.TestInBranch;
import org.apache.ignite.ci.db.TcHelperDb;
import org.apache.ignite.ci.di.scheduler.DirectExecNoWaitSheduler;
import org.apache.ignite.ci.di.scheduler.IScheduler;
import org.apache.ignite.ci.di.scheduler.NoOpSheduler;
+import org.apache.ignite.ci.tcbot.chain.PrChainsProcessorTest;
import org.apache.ignite.ci.tcmodel.changes.ChangesList;
import org.apache.ignite.ci.tcmodel.conf.BuildType;
import org.apache.ignite.ci.tcmodel.conf.Project;
@@ -52,6 +54,8 @@ import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompactedDao;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistSync;
import org.apache.ignite.ci.teamcity.pure.BuildHistoryEmulator;
import org.apache.ignite.ci.teamcity.pure.ITeamcityHttpConnection;
import org.apache.ignite.ci.user.ICredentialsProv;
@@ -369,7 +373,7 @@ public class IgnitedTcInMemoryIntegrationTest {
Injector injector = Guice.createInjector(new AbstractModule() {
@Override protected void configure() {
bind(Ignite.class).toInstance(ignite);
- bind(IStringCompactor.class).to(IgniteStringCompactor.class);
+ bind(IStringCompactor.class).to(IgniteStringCompactor.class).in(new SingletonScope());
}
});
@@ -446,4 +450,99 @@ public class IgnitedTcInMemoryIntegrationTest {
return refBuild;
}
+ @Test
+ public void testRunHistSaveLoad() {
+ Injector injector = Guice.createInjector(new TeamcityIgnitedModule(), new IgniteAndShedulerTestModule());
+
+ injector.getInstance(RunHistCompactedDao.class).init();
+ final IStringCompactor c = injector.getInstance(IStringCompactor.class);
+
+ final String srvId = "apache";
+ final String btId = "RunAll";
+ final String branch = ITeamcity.DEFAULT;
+
+ final PrChainsProcessorTest tst = new PrChainsProcessorTest();
+ tst.initBuildChain(c, btId, branch);
+
+ final Map<Integer, FatBuildCompacted> buildsMap = tst.apacheBuilds();
+
+ final RunHistSync histSync = injector.getInstance(RunHistSync.class);
+ buildsMap.forEach((id, build) -> histSync.saveToHistoryLater(srvId, build));
+
+ final ITeamcityIgnitedProvider inst = injector.getInstance(ITeamcityIgnitedProvider.class);
+ final ITeamcityIgnited srv = inst.server(srvId, Mockito.mock(ICredentialsProv.class));
+ final IRunHistory testRunHist = srv.getTestRunHist(new TestInBranch(PrChainsProcessorTest.TEST_FLAKY_IN_MASTER, branch));
+
+ assertNotNull(testRunHist);
+ assertEquals(0.5, testRunHist.getFailRate(), 0.1);
+
+ final IRunHistory cache1Hist = srv.getSuiteRunHist(new SuiteInBranch(PrChainsProcessorTest.CACHE_1, branch));
+
+ assertNotNull(cache1Hist);
+ assertEquals(1.0, cache1Hist.getFailRate(), 0.1);
+ assertEquals(0.18, cache1Hist.getCriticalFailRate(), 0.05);
+
+ final IRunStat cache1HistAllBranch = srv.getSuiteRunStatAllBranches(PrChainsProcessorTest.CACHE_1);
+
+ assertNotNull(cache1HistAllBranch);
+
+ String printable = cache1HistAllBranch.getFailPercentPrintable();
+ System.err.println(printable);
+ // should be several builds in a separate branch
+ assertEquals(0.5, cache1HistAllBranch.getFailRate(), 0.05);
+ }
+
+
+ @Test
+ public void testHistoryBackgroundUpdateWorks() {
+ Injector injector = Guice.createInjector(new TeamcityIgnitedModule(), new IgniteAndShedulerTestModule());
+
+ injector.getInstance(RunHistCompactedDao.class).init();
+
+ final String srvId = "apache";
+ final String btId = "RunAll";
+ final String branch = ITeamcity.DEFAULT;
+
+ final ITeamcityIgnitedProvider inst = injector.getInstance(ITeamcityIgnitedProvider.class);
+ final ITeamcityIgnited srv = inst.server(srvId, Mockito.mock(ICredentialsProv.class));
+
+ FatBuildDao fatBuildDao = injector.getInstance(FatBuildDao.class);
+ fatBuildDao.init();
+
+ BuildRefDao buildRefDao = injector.getInstance(BuildRefDao.class);
+ buildRefDao.init();
+
+ final IStringCompactor c = injector.getInstance(IStringCompactor.class);
+
+
+ final PrChainsProcessorTest tst = new PrChainsProcessorTest();
+ tst.initBuildChain(c, btId, branch);
+
+ final Map<Integer, FatBuildCompacted> buildsMap = tst.apacheBuilds();
+
+ buildsMap.forEach((id, build) -> {
+ int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
+ fatBuildDao.putFatBuild(srvIdMaskHigh, id, build);
+ buildRefDao.save(srvIdMaskHigh, new BuildRefCompacted(build));
+ });
+
+ final RunHistSync histSync = injector.getInstance(RunHistSync.class);
+ histSync.invokeLaterFindMissingHistory(srvId);
+
+ final IRunHistory testRunHist = srv.getTestRunHist(new TestInBranch(PrChainsProcessorTest.TEST_FLAKY_IN_MASTER, branch));
+
+ assertNotNull(testRunHist);
+ assertEquals(0.5, testRunHist.getFailRate(), 0.1);
+ }
+
+ /**
+ *
+ */
+ private static class IgniteAndShedulerTestModule extends AbstractModule {
+ /** {@inheritDoc} */
+ @Override protected void configure() {
+ bind(Ignite.class).toInstance(ignite);
+ bind(IScheduler.class).to(DirectExecNoWaitSheduler.class).in(new SingletonScope());
+ }
+ }
}
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
index 1cc9156..779b37f 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedMock.java
@@ -18,12 +18,17 @@
package org.apache.ignite.ci.teamcity.ignited;
import com.google.common.base.Preconditions;
+import org.apache.ignite.ci.analysis.TestInBranch;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
import org.jetbrains.annotations.NotNull;
import org.mockito.Mockito;
import java.util.Comparator;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import static org.mockito.ArgumentMatchers.any;
@@ -35,6 +40,9 @@ public class TeamcityIgnitedMock {
@NotNull
public static ITeamcityIgnited getMutableMapTeamcityIgnited(Map<Integer, FatBuildCompacted> builds, IStringCompactor c) {
ITeamcityIgnited tcIgnited = Mockito.mock(ITeamcityIgnited.class);
+ Map<RunHistKey, RunHistCompacted> histCache = new ConcurrentHashMap<>();
+ final int srvId = 0;
+
when(tcIgnited.getFatBuild(anyInt(), any(SyncMode.class)))
.thenAnswer(inv ->
{
@@ -52,7 +60,7 @@ public class TeamcityIgnitedMock {
return builds.values()
.stream()
.filter(fb -> btId.equals(fb.buildTypeId(c)))
- // .filter(fb -> branch.equals(fb.branchName(c)))
+ .filter(fb -> branch.equals(fb.branchName(c)))
.sorted(Comparator.comparing(BuildRefCompacted::id).reversed())
.collect(Collectors.toList());
});
@@ -75,6 +83,60 @@ public class TeamcityIgnitedMock {
.collect(Collectors.toList());
});
+ when(tcIgnited.getTestRunHist(any(TestInBranch.class)))
+ .thenAnswer((inv) -> {
+ final TestInBranch t = inv.getArgument(0);
+ final String name = t.name;
+ final String branch = t.branch;
+
+ // System.out.println("Search history " + name + " in " + branch + ": " );
+
+ if (histCache.isEmpty()) {
+ synchronized (histCache) {
+ if (histCache.isEmpty()) {
+ initHistory(c, histCache, builds, srvId);
+ }
+ }
+ }
+
+ final Integer tstName = c.getStringIdIfPresent(name);
+ if (tstName == null)
+ return null;
+
+ final Integer branchId = c.getStringIdIfPresent(branch);
+ if (branchId == null)
+ return null;
+
+ final RunHistKey key = new RunHistKey(srvId, tstName, branchId);
+
+ final RunHistCompacted runHistCompacted = histCache.get(key);
+
+ System.out.println("Test history " + name + " in " + branch + " => " + runHistCompacted);
+
+ return runHistCompacted;
+ });
+
return tcIgnited;
}
+
+ public static void initHistory(IStringCompactor c, Map<RunHistKey, RunHistCompacted> resHistCache, Map<Integer, FatBuildCompacted> builds, int srvId) {
+ Map<RunHistKey, RunHistCompacted> histCache = new ConcurrentHashMap<>();
+
+ for (FatBuildCompacted build : builds.values()) {
+ if(!build.isFinished(c))
+ continue;
+
+ build.getAllTests().forEach(t -> {
+ RunHistKey histKey = new RunHistKey(srvId, t.testName(), build.branchName());
+
+ final RunHistCompacted hist = histCache.computeIfAbsent(histKey, RunHistCompacted::new);
+
+ Invocation inv = t.toInvocation(c, build);
+
+ hist.addInvocation(inv);
+ });
+ }
+
+ resHistCache.putAll(histCache);
+ }
}
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedProviderMock.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedProviderMock.java
index 298ea46..f9f0eca 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedProviderMock.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedProviderMock.java
@@ -27,7 +27,7 @@ import java.util.concurrent.ConcurrentHashMap;
public class TeamcityIgnitedProviderMock implements ITeamcityIgnitedProvider {
@Inject IStringCompactor compactor;
- Map<String,Map<Integer, FatBuildCompacted>> tcBuildsData = new ConcurrentHashMap<>();
+ private Map<String, Map<Integer, FatBuildCompacted>> tcBuildsData = new ConcurrentHashMap<>();
public void addServer(String srvId, Map<Integer, FatBuildCompacted> apacheBuilds) {
tcBuildsData.put(srvId, apacheBuilds);
@@ -36,6 +36,7 @@ public class TeamcityIgnitedProviderMock implements ITeamcityIgnitedProvider {
@Override
public ITeamcityIgnited server(String srvId, ICredentialsProv prov) {
final Map<Integer, FatBuildCompacted> integerFatBuildCompactedMap = tcBuildsData.get(srvId);
+
return TeamcityIgnitedMock.getMutableMapTeamcityIgnited(integerFatBuildCompactedMap, compactor);
}
}