You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/25 15:25:43 UTC
[ignite-teamcity-bot] branch ignite-9848-load-all-builds updated:
Removed loading of test results using old cache
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-load-all-builds
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9848-load-all-builds by this push:
new 3e6ef42 Removed loading of test results using old cache
3e6ef42 is described below
commit 3e6ef4213250776dcfce1ee6e64687464d1ddf4c
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Oct 25 18:25:12 2018 +0300
Removed loading of test results using old cache
---
...tFailureOccurrences.java => ITestFailures.java} | 8 +++-
.../apache/ignite/ci/analysis/MultBuildRunCtx.java | 47 +++-------------------
.../ci/analysis/MultTestFailureOccurrences.java | 11 +++--
.../ignite/ci/analysis/SingleBuildRunCtx.java | 8 ----
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 25 ++----------
.../model/current/ChainAtServerCurrentStatus.java | 8 ++--
.../ci/web/model/current/SuiteCurrentStatus.java | 16 +++-----
.../ignite/ci/web/model/current/TestFailure.java | 10 ++---
8 files changed, 38 insertions(+), 95 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailureOccurrences.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailures.java
similarity index 85%
rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailureOccurrences.java
rename to ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailures.java
index fb26911..af920b5 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailureOccurrences.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/ITestFailures.java
@@ -18,8 +18,12 @@
package org.apache.ignite.ci.analysis;
import java.util.stream.Stream;
+import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
-public interface ITestFailureOccurrences {
+/**
+ * Multiple test ocurrence
+ */
+public interface ITestFailures {
String getName();
boolean isInvestigated();
@@ -31,4 +35,6 @@ public interface ITestFailureOccurrences {
public int failuresCount();
public long getAvgDurationMs();
+
+ Iterable<TestOccurrenceFull> getOccurrences();
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
index 13d7956..eac6aad 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultBuildRunCtx.java
@@ -61,14 +61,6 @@ public class MultBuildRunCtx implements ISuiteResults {
private final Map<String, MultTestFailureOccurrences> tests = new ConcurrentSkipListMap<>();
/**
- * Mapping for building test occurrence reference to test full results:
- * Map from "Occurrence in build id" test detailed info.
- * Note: only failed tests are loaded here
- */
- @Deprecated
- private Map<String, CompletableFuture<TestOccurrenceFull>> testFullMap = new HashMap<>();
-
- /**
* Statistics for last build.
*/
@Nullable private Statistics stat;
@@ -214,7 +206,7 @@ public class MultBuildRunCtx implements ISuiteResults {
getCriticalFailLastStartedTest().forEach(lastStartedTest ->
builder.append("\t").append(lastStartedTest).append(" (Last started) \n"));
- getFailedTests().map(ITestFailureOccurrences::getName).forEach(
+ getFailedTests().map(ITestFailures::getName).forEach(
name -> {
builder.append("\t").append(name).append("\n");
}
@@ -300,19 +292,19 @@ public class MultBuildRunCtx implements ISuiteResults {
return CollectionUtil.top(logSizeBytes.entrySet().stream(), 3 ,comparing).stream();
}
- public Stream<? extends ITestFailureOccurrences> getTopLongRunning() {
+ public Stream<? extends ITestFailures> getTopLongRunning() {
Stream<MultTestFailureOccurrences> stream = tests.values().stream();
Comparator<MultTestFailureOccurrences> comparing = Comparator.comparing(MultTestFailureOccurrences::getAvgDurationMs);
return CollectionUtil.top(stream, 3, comparing).stream();
}
- public Stream<? extends ITestFailureOccurrences> getFailedTests() {
+ public Stream<? extends ITestFailures> getFailedTests() {
return tests.values().stream().filter(MultTestFailureOccurrences::hasFailedButNotMuted);
}
@Deprecated
- public void addTests(Iterable<TestOccurrence> tests) {
- for (TestOccurrence next : tests) {
+ public void addTests(Iterable<TestOccurrenceFull> tests) {
+ for (TestOccurrenceFull next : tests) {
this.tests.computeIfAbsent(next.name,
k -> new MultTestFailureOccurrences())
.add(next);
@@ -349,23 +341,6 @@ public class MultBuildRunCtx implements ISuiteResults {
}
/**
- * @param testOccurrenceInBuildId, something like: 'id:15666,build:(id:1093907)'
- * @param fullFut
- */
- @Deprecated
- public void addTestInBuildToTestFull(String testOccurrenceInBuildId,
- CompletableFuture<TestOccurrenceFull> fullFut) {
- testFullMap.put(testOccurrenceInBuildId, fullFut);
- }
-
- @Deprecated
- private Optional<TestOccurrenceFull> getFullTest(String testOccurrenceInBuildId) {
- return Optional.ofNullable(testFullMap.get(testOccurrenceInBuildId))
- .flatMap(fut ->
- Optional.ofNullable(fut.isDone() ? FutureUtil.getResultSilent(fut) : null));
- }
-
- /**
* @return aggregation project ID, such as "Ignite_Tests_20"
*/
@Nullable public String projectId() {
@@ -409,14 +384,6 @@ public class MultBuildRunCtx implements ISuiteResults {
return val == null ? 0 : val.intValue();
}
- @Deprecated
- public Stream<TestOccurrenceFull> getFullTests(ITestFailureOccurrences occurrence) {
- return occurrence.getOccurrenceIds()
- .map(this::getFullTest)
- .filter(Optional::isPresent)
- .map(Optional::get);
- }
-
/**
* @return Username's stream for users introduced changes in this commit
*/
@@ -431,14 +398,12 @@ public class MultBuildRunCtx implements ISuiteResults {
Stream<CompletableFuture<?>> stream1 = queuedBuildCount != null ? Stream.of(queuedBuildCount) : Stream.empty();
Stream<CompletableFuture<?>> stream2 = runningBuildCount != null ? Stream.of(runningBuildCount) : Stream.empty();
- Stream<? extends Future<?>> stream3 = testFullMap.values().stream();
-
Stream<? extends Future<?>> stream4 = buildsStream().flatMap(SingleBuildRunCtx::getFutures);
return
concat(
concat(stream1, stream2),
- concat(stream4, stream3));
+ stream4);
}
/**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultTestFailureOccurrences.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultTestFailureOccurrences.java
index a882949..98aa985 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultTestFailureOccurrences.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/MultTestFailureOccurrences.java
@@ -22,9 +22,10 @@ import java.util.Objects;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.stream.Stream;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
+import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
-public class MultTestFailureOccurrences implements ITestFailureOccurrences {
- private final List<TestOccurrence> occurrences = new CopyOnWriteArrayList<>();
+public class MultTestFailureOccurrences implements ITestFailures {
+ private final List<TestOccurrenceFull> occurrences = new CopyOnWriteArrayList<>();
public MultTestFailureOccurrences() {
@@ -66,7 +67,11 @@ public class MultTestFailureOccurrences implements ITestFailureOccurrences {
return stream.findAny().orElse(0L);
}
- public void add(TestOccurrence next) {
+ @Override public Iterable<TestOccurrenceFull> getOccurrences() {
+ return occurrences;
+ }
+
+ public void add(TestOccurrenceFull next) {
if (next.getId() == null)
return;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
index 5456e6a..e695419 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/SingleBuildRunCtx.java
@@ -53,9 +53,6 @@ public class SingleBuildRunCtx implements ISuiteResults {
private List<Change> changes = new ArrayList<>();
- @Deprecated
- private List<TestOccurrence> tests = new ArrayList<>();
-
public SingleBuildRunCtx(Build build,
FatBuildCompacted buildCompacted,
IStringCompactor compactor) {
@@ -159,11 +156,6 @@ public class SingleBuildRunCtx implements ISuiteResults {
return changes;
}
- @Deprecated
- public void setTests(List<TestOccurrence> tests) {
- this.tests = tests;
- }
-
public List<? extends TestOccurrence> getTests() {
return buildCompacted.getTestOcurrences(compactor).getTests();
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index 2febb69..53cf47d 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -23,7 +23,6 @@ import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -46,8 +45,7 @@ import org.apache.ignite.ci.tcmodel.changes.ChangeRef;
import org.apache.ignite.ci.tcmodel.changes.ChangesList;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
+import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnitedProvider;
@@ -60,7 +58,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.google.common.base.Strings.isNullOrEmpty;
-import static org.apache.ignite.ci.db.DbMigrations.TESTS_COUNT_7700;
/**
* Process whole Build Chain, E.g. runAll at particular server, including all builds involved
@@ -72,8 +69,7 @@ public class BuildChainProcessor {
/** TC REST updates pool. */
@Inject private TcUpdatePool tcUpdatePool;
- @Inject private ITeamcityIgnitedProvider teamcityIgnitedProvider;
-
+ /** Compactor. */
@Inject private IStringCompactor compactor;
/**
@@ -216,22 +212,9 @@ public class BuildChainProcessor {
}
if (build.testOccurrences != null && !build.isComposite()) {
- List<TestOccurrence> tests
- = teamcity.getTests(build.testOccurrences.href + TESTS_COUNT_7700).getTests();
-
- ctx.setTests(tests);
-
- mCtx.addTests(tests);
-
- for (TestOccurrence next : tests) {
- if (next.href != null && next.isFailedTest()) {
- CompletableFuture<TestOccurrenceFull> testFullFut = teamcity.getTestFull(next.href);
+ TestOccurrencesFull tests = buildCompacted.getTestOcurrences(compactor);
- String testInBuildId = next.getId();
-
- mCtx.addTestInBuildToTestFull(testInBuildId, testFullFut);
- }
- }
+ mCtx.addTests(tests.getTests());
}
if (build.statisticsRef != null)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/ChainAtServerCurrentStatus.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/ChainAtServerCurrentStatus.java
index 43ed1c8..8aacb86 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/ChainAtServerCurrentStatus.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/ChainAtServerCurrentStatus.java
@@ -27,7 +27,7 @@ import javax.annotation.Nullable;
import org.apache.ignite.ci.ITcAnalytics;
import org.apache.ignite.ci.ITeamcity;
import org.apache.ignite.ci.analysis.FullChainRunCtx;
-import org.apache.ignite.ci.analysis.ITestFailureOccurrences;
+import org.apache.ignite.ci.analysis.ITestFailures;
import org.apache.ignite.ci.analysis.MultBuildRunCtx;
import org.apache.ignite.ci.util.CollectionUtil;
import org.apache.ignite.internal.util.typedef.T2;
@@ -110,16 +110,16 @@ public class ChainAtServerCurrentStatus {
webToHist = buildWebLink(teamcity, ctx);
webToBuild = buildWebLinkToBuild(teamcity, ctx);
- Stream<T2<MultBuildRunCtx, ITestFailureOccurrences>> allLongRunning = ctx.suites().stream().flatMap(
+ Stream<T2<MultBuildRunCtx, ITestFailures>> allLongRunning = ctx.suites().stream().flatMap(
suite -> suite.getTopLongRunning().map(t -> new T2<>(suite, t))
);
- Comparator<T2<MultBuildRunCtx, ITestFailureOccurrences>> durationComp
+ Comparator<T2<MultBuildRunCtx, ITestFailures>> durationComp
= Comparator.comparing((pair) -> pair.get2().getAvgDurationMs());
CollectionUtil.top(allLongRunning, 3, durationComp).forEach(
pairCtxAndOccur -> {
MultBuildRunCtx suite = pairCtxAndOccur.get1();
- ITestFailureOccurrences longRunningOccur = pairCtxAndOccur.get2();
+ ITestFailures longRunningOccur = pairCtxAndOccur.get2();
TestFailure failure = createOrrucForLongRun(teamcity, suite, tcAnalytics, longRunningOccur, baseBranchTc);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
index 9498654..b693f73 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
@@ -31,7 +31,7 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.ignite.ci.ITcAnalytics;
import org.apache.ignite.ci.ITeamcity;
-import org.apache.ignite.ci.analysis.ITestFailureOccurrences;
+import org.apache.ignite.ci.analysis.ITestFailures;
import org.apache.ignite.ci.analysis.MultBuildRunCtx;
import org.apache.ignite.ci.analysis.RunStat;
import org.apache.ignite.ci.analysis.SuiteInBranch;
@@ -140,9 +140,9 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
webToHistBaseBranch = buildWebLink(teamcity, suite, baseBranch);
webToBuild = buildWebLinkToBuild(teamcity, suite);
- Stream<? extends ITestFailureOccurrences> tests = suite.getFailedTests();
+ Stream<? extends ITestFailures> tests = suite.getFailedTests();
if (tcAnalytics != null) {
- Function<ITestFailureOccurrences, Float> function = foccur -> {
+ Function<ITestFailures, Float> function = foccur -> {
TestInBranch branch = new TestInBranch(foccur.getName(), failRateNormalizedBranch);
RunStat apply = tcAnalytics.getTestRunStatProvider().apply(branch);
@@ -153,10 +153,8 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
}
tests.forEach(occurrence -> {
- Stream<TestOccurrenceFull> stream = suite.getFullTests(occurrence);
-
final TestFailure failure = new TestFailure();
- failure.initFromOccurrence(occurrence, stream, teamcity, suite.projectId(), suite.branchName(), baseBranch);
+ failure.initFromOccurrence(occurrence, teamcity, suite.projectId(), suite.branchName(), baseBranch);
if (tcAnalytics != null)
failure.initStat(tcAnalytics.getTestRunStatProvider(), failRateNormalizedBranch, curBranchNormalized);
@@ -271,13 +269,11 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
@NotNull public static TestFailure createOrrucForLongRun(@Nonnull ITeamcity teamcity,
@Nonnull MultBuildRunCtx suite,
@Nullable final ITcAnalytics tcAnalytics,
- final ITestFailureOccurrences occurrence,
+ final ITestFailures occurrence,
@Nullable final String failRateBranch) {
final TestFailure failure = new TestFailure();
- Stream<TestOccurrenceFull> stream = suite.getFullTests(occurrence);
-
- failure.initFromOccurrence(occurrence, stream, teamcity, suite.projectId(), suite.branchName(), failRateBranch);
+ failure.initFromOccurrence(occurrence, teamcity, suite.projectId(), suite.branchName(), failRateBranch);
if (tcAnalytics != null) {
failure.initStat(tcAnalytics.getTestRunStatProvider(),
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java
index 7e49ec0..dbee4f7 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/TestFailure.java
@@ -24,17 +24,15 @@ import java.util.List;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.ignite.ci.ITeamcity;
-import org.apache.ignite.ci.analysis.ITestFailureOccurrences;
+import org.apache.ignite.ci.analysis.ITestFailures;
import org.apache.ignite.ci.analysis.RunStat;
import org.apache.ignite.ci.analysis.TestInBranch;
import org.apache.ignite.ci.issue.EventTemplates;
import org.apache.ignite.ci.issue.ProblemRef;
import org.apache.ignite.ci.logs.LogMsgToWarn;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.web.model.hist.FailureSummary;
import org.apache.ignite.ci.web.model.hist.TestHistory;
import org.jetbrains.annotations.NotNull;
@@ -94,14 +92,12 @@ import static org.apache.ignite.ci.web.model.current.SuiteCurrentStatus.branchFo
/**
* @param failure
- * @param testFullOpt all related full test ocurrences
* @param teamcity
* @param projectId
* @param branchName
* @param baseBranchName base branch name (e.g. master).
*/
- public void initFromOccurrence(@Nonnull final ITestFailureOccurrences failure,
- @Nonnull final Stream<TestOccurrenceFull> testFullOpt,
+ public void initFromOccurrence(@Nonnull final ITestFailures failure,
@Nonnull final ITeamcity teamcity,
@Nullable final String projectId,
@Nullable final String branchName,
@@ -124,7 +120,7 @@ import static org.apache.ignite.ci.web.model.current.SuiteCurrentStatus.branchFo
testName = testComps[testComps.length - 2] + "." + testComps[testComps.length - 1];
}
- testFullOpt.forEach(full -> {
+ failure.getOccurrences().forEach(full -> {
String details = full.details;
if (details != null) {