You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ignite.apache.org by GitBox <gi...@apache.org> on 2018/10/02 15:28:37 UTC
[GitHub] dspavlov closed pull request #20: Ignite 9645
dspavlov closed pull request #20: Ignite 9645
URL: https://github.com/apache/ignite-teamcity-bot/pull/20
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 9481318..de34613 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -42,6 +42,7 @@
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrences;
+import org.apache.ignite.ci.tcmodel.result.tests.TestRef;
import org.apache.ignite.ci.util.Base64Util;
import org.jetbrains.annotations.NotNull;
@@ -152,6 +153,8 @@ default Build getBuild(int id) {
TestOccurrences getTests(String href, String normalizedBranch);
+ TestOccurrences getFailedUnmutedTests(String href, String normalizedBranch);
+
Statistics getBuildStatistics(String href);
CompletableFuture<TestOccurrenceFull> getTestFull(String href);
@@ -160,6 +163,8 @@ default Build getBuild(int id) {
ChangesList getChangesList(String href);
+ CompletableFuture<TestRef> getTestRef(TestOccurrence occurrence);
+
/**
* List of build's related issues.
*
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index bbf96a2..9def938 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -72,6 +72,7 @@
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrences;
+import org.apache.ignite.ci.tcmodel.result.tests.TestRef;
import org.apache.ignite.ci.util.CacheUpdateUtil;
import org.apache.ignite.ci.util.CollectionUtil;
import org.apache.ignite.ci.util.ObjectInterner;
@@ -103,6 +104,7 @@
private static final String BUILD_HIST_FINISHED = "buildHistFinished";
private static final String BUILD_HIST_FINISHED_OR_FAILED = "buildHistFinishedOrFailed";
public static final String BOT_DETECTED_ISSUES = "botDetectedIssues";
+ public static final String TEST_REFS = "testRefs";
//todo need separate cache or separate key for 'execution time' because it is placed in statistics
private static final String BUILDS_FAILURE_RUN_STAT = "buildsFailureRunStat";
@@ -124,6 +126,11 @@
*/
private ConcurrentMap<String, CompletableFuture<TestOccurrenceFull>> testOccFullFutures = new ConcurrentHashMap<>();
+ /**
+ * cached loads of test refs.
+ */
+ private ConcurrentMap<String, CompletableFuture<TestRef>> testRefsFutures = new ConcurrentHashMap<>();
+
/** cached running builds for branch. */
private ConcurrentMap<String, Expirable<List<BuildRef>>> queuedBuilds = new ConcurrentHashMap<>();
@@ -217,6 +224,13 @@ public void init(String serverId) {
return getOrCreateCacheV2(ignCacheNme(TEST_FULL));
}
+ /**
+ * @return {@link TestRef} instances cache, 32 parts.
+ */
+ private IgniteCache<String, TestRef> testRefsCache() {
+ return getOrCreateCacheV2(ignCacheNme(TEST_REFS));
+ }
+
/**
* @return Build {@link ProblemOccurrences} instances cache, 32 parts.
*/
@@ -231,7 +245,6 @@ public void init(String serverId) {
return getOrCreateCacheV2(ignCacheNme(BUILD_STATISTICS));
}
-
/**
* @return Build history: {@link BuildRef} lists cache, 32 parts, transactional.
*/
@@ -824,6 +837,11 @@ private void registerCriticalBuildProblemInStat(Build build, ProblemOccurrences
hrefIgnored -> teamcity.getTests(href, normalizedBranch));
}
+ @AutoProfiling
+ @Override public TestOccurrences getFailedUnmutedTests(String href, String normalizedBranch) {
+ return getTests(href + ",muted:false,status:FAILURE", normalizedBranch);
+ }
+
private void addTestOccurrencesToStat(TestOccurrences val) {
for (TestOccurrence next : val.getTests())
addTestOccurrenceToStat(next, ITeamcity.DEFAULT, null);
@@ -871,6 +889,23 @@ private void addTestOccurrencesToStat(TestOccurrences val) {
});
}
+ @Override
+ @AutoProfiling
+ public CompletableFuture<TestRef> getTestRef(TestOccurrence testOccurrence) {
+ return CacheUpdateUtil.loadAsyncIfAbsent(
+ testRefsCache(),
+ testOccurrence.name,
+ testRefsFutures,
+ name -> {
+ try {
+ return teamcity.getTestRef(testOccurrence);
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
/** {@inheritDoc} */
@AutoProfiling
@Override public Change getChange(String href) {
@@ -914,9 +949,10 @@ private void addTestOccurrencesToStat(TestOccurrences val) {
});
}
+ /** {@inheritDoc} */
@AutoProfiling
@Override public IssuesUsagesList getIssuesUsagesList(String href) {
- IssuesUsagesList issuesUsages = loadIfAbsentV2(ISSUES_USAGES_LIST, href, href1 -> {
+ IssuesUsagesList issuesUsages = loadIfAbsentV2(ISSUES_USAGES_LIST, href, href1 -> {
try {
return teamcity.getIssuesUsagesList(href1);
}
@@ -1061,7 +1097,6 @@ private void migrateTestOneOcurrToAddToLatest(TestOccurrence next) {
k -> teamcity.analyzeBuildLog(buildId, ctx));
}
-
@AutoProfiling
@Override
public String getThreadDumpCached(Integer buildId) {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
index 070f4b2..510e5a0 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
@@ -70,8 +70,10 @@
import org.apache.ignite.ci.tcmodel.result.issues.IssuesUsagesList;
import org.apache.ignite.ci.tcmodel.result.problems.ProblemOccurrences;
import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
+import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrences;
+import org.apache.ignite.ci.tcmodel.result.tests.TestRef;
import org.apache.ignite.ci.tcmodel.user.User;
import org.apache.ignite.ci.tcmodel.user.Users;
import org.apache.ignite.ci.util.*;
@@ -552,6 +554,22 @@ public ProblemOccurrences getProblems(Build build) {
return supplyAsync(() -> getJaxbUsingHref(href, TestOccurrenceFull.class), executor);
}
+ @AutoProfiling
+ @Override public TestOccurrences getFailedUnmutedTests(String href, String normalizedBranch) {
+ return getTests(href + ",muted:false,status:FAILURE", normalizedBranch);
+ }
+
+ @Override
+ @AutoProfiling
+ public CompletableFuture<TestRef> getTestRef(TestOccurrence testOccurrence) {
+ return supplyAsync(() -> {
+ if (testOccurrence.href == null) {
+ return new TestRef();
+ }
+ return getJaxbUsingHref(testOccurrence.href, TestOccurrenceFull.class).test;
+ }, executor);
+ }
+
/** {@inheritDoc} */
@AutoProfiling
@Override public Change getChange(String href) {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildsHistory.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildsHistory.java
new file mode 100644
index 0000000..97d4e61
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/BuildsHistory.java
@@ -0,0 +1,199 @@
+package org.apache.ignite.ci.web.model.current;
+
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import javax.servlet.ServletContext;
+import org.apache.ignite.ci.BuildChainProcessor;
+import org.apache.ignite.ci.IAnalyticsEnabledTeamcity;
+import org.apache.ignite.ci.ITcHelper;
+import org.apache.ignite.ci.tcmodel.hist.BuildRef;
+import org.apache.ignite.ci.tcmodel.result.Build;
+import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrence;
+import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrences;
+import org.apache.ignite.ci.tcmodel.result.tests.TestRef;
+import org.apache.ignite.ci.user.ICredentialsProv;
+import org.apache.ignite.ci.web.CtxListener;
+import org.apache.ignite.ci.web.rest.parms.FullQueryParams;
+
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static org.apache.ignite.ci.web.rest.build.GetBuildTestFailures.BUILDS_STATISTICS_SUMMARY_CACHE_NAME;
+
+public class BuildsHistory {
+ private String srvId;
+
+ private String buildTypeId;
+
+ private String branchName;
+
+ private Date sinceDateFilter;
+
+ private Date untilDateFilter;
+
+ public List<BuildStatisticsSummary> buildsStatistics = new ArrayList<>();
+
+ public Map<String, List<TestRef>> mergedTestsBySuites = new HashMap<>();
+
+ public void initialize(ICredentialsProv prov, ServletContext context) {
+ ITcHelper tcHelper = CtxListener.getTcHelper(context);
+
+ IAnalyticsEnabledTeamcity teamcity = tcHelper.server(srvId, prov);
+ int[] finishedBuildsIds = teamcity.getBuildNumbersFromHistory(buildTypeId, branchName,
+ sinceDateFilter, untilDateFilter);
+
+ initBuildsStatistics(teamcity, prov, context, finishedBuildsIds);
+
+ initBuildsMergedFailedTests(teamcity, finishedBuildsIds);
+ }
+
+ private void initBuildsStatistics(IAnalyticsEnabledTeamcity teamcity, ICredentialsProv prov, ServletContext context, int[] buildIds) {
+ for (int buildId : buildIds) {
+ FullQueryParams buildParams = new FullQueryParams();
+
+ buildParams.setBuildId(buildId);
+
+ buildParams.setBranch(branchName);
+
+ buildParams.setServerId(srvId);
+
+ BuildStatisticsSummary buildsStatistic = CtxListener.getBackgroundUpdater(context).get(
+ BUILDS_STATISTICS_SUMMARY_CACHE_NAME, prov, buildParams,
+ (k) -> {
+ BuildStatisticsSummary stat = new BuildStatisticsSummary(buildId);
+
+ stat.initialize(teamcity);
+
+ return stat;
+ }, false);
+
+ if (buildsStatistic != null && !buildsStatistic.isFakeStub)
+ buildsStatistics.add(buildsStatistic);
+ }
+ }
+
+ private void initBuildsMergedFailedTests(IAnalyticsEnabledTeamcity teamcity, int[] buildIds) {
+ Map<String, Map<String, CompletableFuture<TestRef>>> mergedTestsFutures = new HashMap<>();
+
+ for (int buildId : buildIds)
+ mergeBuildTestsFuturesWith(mergedTestsFutures, teamcity, buildId);
+
+ mergedTestsFutures.entrySet().forEach(entry -> {
+ List<TestRef> tests =
+ mergedTestsBySuites.computeIfAbsent(entry.getKey(), k -> new ArrayList<>());
+
+ entry.getValue().values().forEach( v -> {
+ try {
+ tests.add(v.get(1, TimeUnit.MINUTES));
+ } catch (TimeoutException e){
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ });
+ }
+
+ private void mergeBuildTestsFuturesWith(Map<String, Map<String, CompletableFuture<TestRef>>> testFutures,
+ IAnalyticsEnabledTeamcity teamcity, int buildId) {
+ Build build = teamcity.getBuild(teamcity.getBuildHrefById(buildId));
+
+ if (build == null || build.isFakeStub() || build.isSuccess() )
+ return;
+
+ if (build.isComposite()) {
+ for (BuildRef buildRef : build.getSnapshotDependenciesNonNull()) {
+ if (buildRef.isSuccess())
+ continue;
+
+ mergeBuildTestsFuturesWith(testFutures, teamcity, buildRef.getId());
+ }
+
+ return;
+ }
+
+ if (build.testOccurrences == null || build.testOccurrences.href == null)
+ return;
+
+ TestOccurrences testOccurrences = teamcity.getFailedUnmutedTests(build.testOccurrences.href,
+ BuildChainProcessor.normalizeBranch(build.branchName));
+
+ Map<String, CompletableFuture<TestRef>> map = testFutures.computeIfAbsent(build.buildTypeId,
+ k -> new HashMap<>());
+
+ for (TestOccurrence testOccurrence : testOccurrences.getTests())
+ map.computeIfAbsent(testOccurrence.getName(), k -> teamcity.getTestRef(testOccurrence));
+ }
+
+ public BuildsHistory(Builder builder) {
+ this.srvId = builder.srvId;
+
+ this.buildTypeId = builder.buildTypeId;
+
+ this.branchName = builder.branchName;
+
+ this.sinceDateFilter = builder.sinceDate;
+
+ this.untilDateFilter = builder.untilDate;
+ }
+
+ public static class Builder {
+ private String srvId = "apache";
+
+ private String buildTypeId = "IgniteTests24Java8_RunAll";
+
+ private String branchName = "refs/heads/master";
+
+ private Date sinceDate = null;
+
+ private Date untilDate = null;
+
+ private DateFormat dateFormat = new SimpleDateFormat("ddMMyyyyHHmmss");
+
+ public Builder server(String srvId) {
+ if (!isNullOrEmpty(srvId))
+ this.srvId = srvId;
+
+ return this;
+ }
+
+ public Builder buildType(String buildType) {
+ if (!isNullOrEmpty(buildType))
+ this.buildTypeId = buildType;
+
+ return this;
+ }
+
+ public Builder branch(String branchName) {
+ if (!isNullOrEmpty(branchName))
+ this.branchName = branchName;
+
+ return this;
+ }
+
+ public Builder sinceDate(String sinceDate) throws ParseException {
+ if (!isNullOrEmpty(sinceDate))
+ this.sinceDate = dateFormat.parse(sinceDate);
+
+ return this;
+ }
+
+ public Builder untilDate(String untilDate) throws ParseException {
+ if (!isNullOrEmpty(untilDate))
+ this.untilDate = dateFormat.parse(untilDate);
+
+ return this;
+ }
+
+ public BuildsHistory build() {
+ return new BuildsHistory(this);
+ }
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java
index 6b63324..508fe8d 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/build/GetBuildTestFailures.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ci.web.rest.build;
+import java.text.ParseException;
import org.apache.ignite.ci.BuildChainProcessor;
import org.apache.ignite.ci.IAnalyticsEnabledTeamcity;
import org.apache.ignite.ci.ITcHelper;
@@ -26,6 +27,8 @@
import org.apache.ignite.ci.analysis.mode.ProcessLogsMode;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.user.ICredentialsProv;
+import org.apache.ignite.ci.web.model.current.BuildStatisticsSummary;
+import org.apache.ignite.ci.web.model.current.BuildsHistory;
import org.apache.ignite.ci.web.BackgroundUpdater;
import org.apache.ignite.ci.web.CtxListener;
import org.apache.ignite.ci.web.model.current.BuildStatisticsSummary;
@@ -49,11 +52,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Optional;
-import java.util.Date;
import java.util.concurrent.atomic.AtomicInteger;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import static com.google.common.base.Strings.isNullOrEmpty;
@@ -158,74 +157,22 @@ public TestFailuresSummary getBuildTestFails(
@GET
@Path("history")
- public List<BuildStatisticsSummary> getBuildsHistory(
+ public BuildsHistory getBuildsHistory(
@Nullable @QueryParam("server") String server,
@Nullable @QueryParam("buildType") String buildType,
@Nullable @QueryParam("branch") String branch,
@Nullable @QueryParam("sinceDate") String sinceDate,
- @Nullable @QueryParam("untilDate") String untilDate)
- throws ServiceUnauthorizedException {
- String srvId = isNullOrEmpty(server) ? "apache" : server;
- String buildTypeId = isNullOrEmpty(buildType) ? "IgniteTests24Java8_RunAll" : buildType;
- String branchName = isNullOrEmpty(branch) ? "refs/heads/master" : branch;
- Date sinceDateFilter = isNullOrEmpty(sinceDate) ? null : dateParse(sinceDate);
- Date untilDateFilter = isNullOrEmpty(untilDate) ? null : dateParse(untilDate);
-
- final BackgroundUpdater updater = CtxListener.getBackgroundUpdater(context);
-
- final ITcHelper tcHelper = CtxListener.getTcHelper(context);
-
- final ICredentialsProv prov = ICredentialsProv.get(req);
-
- IAnalyticsEnabledTeamcity teamcity = tcHelper.server(srvId, prov);
-
- int[] finishedBuilds = teamcity.getBuildNumbersFromHistory(buildTypeId, branchName, sinceDateFilter, untilDateFilter);
-
- List<BuildStatisticsSummary> buildsStatistics = new ArrayList<>();
-
- for (int i = 0; i < finishedBuilds.length; i++) {
- int buildId = finishedBuilds[i];
-
- FullQueryParams param = new FullQueryParams();
- param.setBuildId(buildId);
- param.setBranch(branchName);
- param.setServerId(srvId);
-
- BuildStatisticsSummary buildsStatistic = updater.get(
- BUILDS_STATISTICS_SUMMARY_CACHE_NAME, prov, param,
- (k) -> getBuildStatisticsSummaryNoCache(srvId, buildId), false);
-
- if (!buildsStatistic.isFakeStub)
- buildsStatistics.add(buildsStatistic);
- }
-
- return buildsStatistics;
- }
-
- private Date dateParse(String date){
- DateFormat dateFormat = new SimpleDateFormat("ddMMyyyyHHmmss");
-
- try {
- return dateFormat.parse(date);
- }
- catch (ParseException e) {
- return null;
- }
- }
-
- private BuildStatisticsSummary getBuildStatisticsSummaryNoCache(String server, int buildId) {
- String srvId = isNullOrEmpty(server) ? "apache" : server;
-
- final ITcHelper tcHelper = CtxListener.getTcHelper(context);
-
- final ICredentialsProv creds = ICredentialsProv.get(req);
-
- IAnalyticsEnabledTeamcity teamcity = tcHelper.server(srvId, creds);
-
- BuildStatisticsSummary stat = new BuildStatisticsSummary(buildId);
-
- stat.initialize(teamcity);
-
- return stat;
+ @Nullable @QueryParam("untilDate") String untilDate) throws ParseException {
+ BuildsHistory buildsHistory = new BuildsHistory.Builder()
+ .branch(branch)
+ .server(server)
+ .buildType(buildType)
+ .sinceDate(sinceDate)
+ .untilDate(untilDate)
+ .build();
+
+ buildsHistory.initialize(ICredentialsProv.get(req), context);
+
+ return buildsHistory;
}
}
diff --git a/ignite-tc-helper-web/src/main/webapp/comparison.html b/ignite-tc-helper-web/src/main/webapp/comparison.html
index 5efe4e3..03cb176 100644
--- a/ignite-tc-helper-web/src/main/webapp/comparison.html
+++ b/ignite-tc-helper-web/src/main/webapp/comparison.html
@@ -304,7 +304,7 @@
url: 'rest/build/history?sinceDate=' + sinceDate.format("DDMMYYYY") +
'000001&untilDate=' + untilDate.format("DDMMYYYY") + '235959',
success: function (result) {
- printStatistics(num, result, sinceDate, untilDate);
+ printStatistics(num, result.buildsStatistics, sinceDate, untilDate);
},
error: showErrInLoadStatus
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services