You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/08/01 15:05:26 UTC
[ignite-teamcity-bot] branch master updated: Tracked branch list
now contains number of suites running and queued
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 3f55641 Tracked branch list now contains number of suites running and queued
3f55641 is described below
commit 3f55641bb2aabc0e4f77455ede46c4efa664c9f7
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Aug 1 18:05:11 2019 +0300
Tracked branch list now contains number of suites running and queued
---
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
.../rest/tracked/GetTrackedBranchTestResults.java | 17 ++++++
ignite-tc-helper-web/src/main/webapp/guard.html | 35 +++++++++++++
.../tracked/IDetailedStatusForTrackedBranch.java | 7 +++
.../tracked/TrackedBranchChainsProcessor.java | 60 ++++++++++++++++++++++
.../tcbot/engine/ui/GuardBranchStatusUi.java | 58 +++++++++++++++++++++
.../apache/ignite/tcignited/ITeamcityIgnited.java | 4 ++
.../ignite/tcignited/TeamcityIgnitedImpl.java | 8 ++-
.../ignite/tcignited/history/HistoryCollector.java | 44 ++++++++--------
9 files changed, 210 insertions(+), 25 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
index b1ce2f9..a9170d3 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
@@ -28,7 +28,7 @@ package org.apache.ignite.ci.web.model;
public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot";
/** TC Bot Version. */
- public static final String VERSION = "20190731";
+ public static final String VERSION = "20190801";
/** Java version, where Web App is running. */
public String javaVer;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java
index 1ff6d7c..3b87184 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java
@@ -18,7 +18,10 @@
package org.apache.ignite.ci.web.rest.tracked;
import com.google.inject.Injector;
+import java.util.List;
+import java.util.Objects;
import java.util.Set;
+import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
@@ -38,6 +41,7 @@ import org.apache.ignite.tcbot.engine.tracked.DisplayMode;
import org.apache.ignite.tcbot.engine.tracked.IDetailedStatusForTrackedBranch;
import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor;
import org.apache.ignite.tcbot.engine.ui.DsSummaryUi;
+import org.apache.ignite.tcbot.engine.ui.GuardBranchStatusUi;
import org.apache.ignite.tcbot.engine.ui.UpdateInfo;
import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
import org.apache.ignite.tcignited.SyncMode;
@@ -220,4 +224,17 @@ public class GetTrackedBranchTestResults {
.getInstance(TcBotTriggerAndSignOffService.class)
.getMutes(srvCode, projectId, creds);
}
+
+ @GET
+ @Path("summary")
+ public List<GuardBranchStatusUi> getIdsIfAccessible() {
+ ITcBotUserCreds prov = ITcBotUserCreds.get(req);
+ Injector injector = CtxListener.getInjector(ctx);
+ ITcBotConfig cfg = injector.getInstance(ITcBotConfig.class);
+ IDetailedStatusForTrackedBranch status = injector.getInstance(IDetailedStatusForTrackedBranch.class);
+
+ return cfg.getTrackedBranches().branchesStream()
+ .map(bt -> status.getBranchSummary(bt.name(), prov)).filter(Objects::nonNull)
+ .collect(Collectors.toList());
+ }
}
diff --git a/ignite-tc-helper-web/src/main/webapp/guard.html b/ignite-tc-helper-web/src/main/webapp/guard.html
index 0078687..8c8d303 100644
--- a/ignite-tc-helper-web/src/main/webapp/guard.html
+++ b/ignite-tc-helper-web/src/main/webapp/guard.html
@@ -33,15 +33,50 @@ function loadData() {
success: function(result) {
$("#loadStatus").html("");
showTrackedBranchesLinks(result);
+
+ $.ajax({
+ url: "rest/tracked/summary",
+ success: function(result) {
+ $("#loadStatus").html("");
+ showTrackedBranchesLinks2(result);
+ },
+ error: showErrInLoadStatus
+ });
},
error: showErrInLoadStatus
});
+
+}
+
+function showTrackedBranchesLinks2(result) {
+ var res = "<table >";
+ res+="<tr><td>Branch name</td><td>Report</td><td>Merged 10</td><td>Finished last day</td><td>Running</td><td>Queued</td></tr>";
+ for (var i = 0; i < result.length; i++) {
+ let summary = result[i];
+ var id = summary.name;
+ res += "<tr>";
+ res += "<td align='right'>" + id + ":</td>";
+ res += "<td>" + "<a href='current.html?branch=" + id + "'><button>Latest (" + id + ")</button></a>" + "</td>";
+ res += "<td>" + " <a href='current.html?branch=" + id + "&count=10'><button title='Failures from last 10 runs merged'>History (" + id + ")</button></a>" + "</td>";
+
+ res += "<td>" + JSON.stringify(summary.finishedLastDay)+ "</td>";
+ res += "<td>" + JSON.stringify(summary.runningList)+ "</td>";
+ res += "<td>" + JSON.stringify(summary.queuedList)+ "</td>";
+ res += "</tr>";
+ }
+
+ res += "</table>";
+
+
+ $("#branchesStatus").html(res);
}
function showTrackedBranchesLinks(result) {
var res = "<table>";
+ res+="<tr><td>Branch name</td><td>Report</td><td>Merged 10</td><td>Finished last day</td><td>Running</td><td>Queued</td></tr>";
+
for (var i = 0; i < result.length; i++) {
var id = result[i];
res += "<tr>";
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java
index 4edad4a..1ba1e7c 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java
@@ -19,6 +19,7 @@ package org.apache.ignite.tcbot.engine.tracked;
import javax.annotation.Nullable;
import org.apache.ignite.tcbot.engine.chain.SortOption;
import org.apache.ignite.tcbot.engine.ui.DsSummaryUi;
+import org.apache.ignite.tcbot.engine.ui.GuardBranchStatusUi;
import org.apache.ignite.tcignited.SyncMode;
import org.apache.ignite.tcignited.creds.ICredentialsProv;
@@ -52,5 +53,11 @@ public interface IDetailedStatusForTrackedBranch {
@Nullable SortOption sortOption,
int maxDurationSec);
+ /**
+ * @param name Name.
+ * @param prov Prov.
+ */
+ public GuardBranchStatusUi getBranchSummary(String name, ICredentialsProv prov);
+
// * @param baseTrackedBranch Branch tracked branch in Bot, has a priority if both TC & Bot branches (baseBranchForTcParm) present.
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java
index 56d266a..0cbab4e 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java
@@ -17,14 +17,17 @@
package org.apache.ignite.tcbot.engine.tracked;
import com.google.common.base.Strings;
+import java.time.Duration;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.inject.Inject;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.tcbot.common.conf.IBuildParameterSpec;
import org.apache.ignite.tcbot.common.conf.IParameterValueSpec;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
@@ -36,8 +39,10 @@ import org.apache.ignite.tcbot.engine.chain.ProcessLogsMode;
import org.apache.ignite.tcbot.engine.chain.SortOption;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.conf.ITrackedBranch;
+import org.apache.ignite.tcbot.engine.conf.ITrackedChain;
import org.apache.ignite.tcbot.engine.ui.DsChainUi;
import org.apache.ignite.tcbot.engine.ui.DsSummaryUi;
+import org.apache.ignite.tcbot.engine.ui.GuardBranchStatusUi;
import org.apache.ignite.tcbot.engine.ui.LrTestsFullSummaryUi;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
@@ -166,6 +171,61 @@ public class TrackedBranchChainsProcessor implements IDetailedStatusForTrackedBr
return res;
}
+ @Override public GuardBranchStatusUi getBranchSummary(String name, ICredentialsProv prov) {
+ ITrackedBranch tb = tcBotCfg.getTrackedBranches().getBranchMandatory(name);
+ List<ITrackedChain> accessibleChains =
+ tb.chainsStream()
+ .filter(chain -> tcIgnitedProv.hasAccess(chain.serverCode(), prov))
+ .collect(Collectors.toList());
+
+ if (accessibleChains == null)
+ return null;
+
+ int ageDays = 1;
+ long minStartTime = System.currentTimeMillis() - Duration.ofDays(ageDays).toMillis();
+
+ GuardBranchStatusUi statusUi = new GuardBranchStatusUi();
+ statusUi.setName(tb.name());
+
+ for (ITrackedChain chain : accessibleChains) {
+ String srvCodeOrAlias = chain.serverCode();
+ ITeamcityIgnited tcIgn = tcIgnitedProv.server(srvCodeOrAlias, prov);
+
+ List<BuildRefCompacted> hist = tcIgn.getAllBuildsCompacted(chain.tcSuiteId(), chain.tcBranch());
+
+ AtomicInteger finished = new AtomicInteger();
+ AtomicInteger running = new AtomicInteger();
+ AtomicInteger queued = new AtomicInteger();
+
+ hist.stream()
+ .filter(ref -> !ref.isFakeStub())
+ .filter(t -> !t.isCancelled(compactor))
+ .peek(ref -> {
+ if (ref.isRunning(compactor))
+ running.incrementAndGet();
+ else if (ref.isQueued(compactor))
+ queued.incrementAndGet();
+ })
+ .filter(ref -> ref.isFinished(compactor))
+ .filter(ref -> {
+ Integer borderId = tcIgn.getBorderForAgeForBuildId(ageDays);
+ return borderId == null || ref.id() >= borderId;
+ })
+ .filter(ref -> {
+ Long startTime = tcIgn.getBuildStartTime(ref.id());
+
+ return startTime != null && startTime > minStartTime;
+ })
+ .forEach(ref -> {
+ finished.incrementAndGet();
+ });
+
+ statusUi.addSuiteRunStat(finished.get(), running.get(), queued.get());
+ }
+
+ return statusUi;
+ }
+
/**
* Collects data about all long-running tests (run time more than one minute) within one transfer object.
*
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/GuardBranchStatusUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/GuardBranchStatusUi.java
new file mode 100644
index 0000000..fd3378a
--- /dev/null
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/GuardBranchStatusUi.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.tcbot.engine.ui;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Branch summary to be shown at guard page, listing
+ */
+public class GuardBranchStatusUi {
+ private String name;
+
+ private List<Integer> finishedLastDay = new ArrayList<>();
+ private List<Integer> runningList = new ArrayList<>();
+ private List<Integer> queuedList = new ArrayList<>();
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public List<Integer> getFinishedLastDay() {
+ return finishedLastDay;
+ }
+
+ public void addSuiteRunStat(int finished, int running, int queued) {
+ finishedLastDay.add(finished);
+ runningList.add(running);
+ queuedList.add(queued);
+ }
+
+
+ public List<Integer> getRunningList() {
+ return runningList;
+ }
+
+ public List<Integer> getQueuedList() {
+ return queuedList;
+ }
+}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
index 17f82bf..e68fd46 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java
@@ -259,4 +259,8 @@ public interface ITeamcityIgnited {
* should be re-synced.
*/
public void actualizeRecentBuildRefs();
+
+ public Long getBuildStartTime(int buildId);
+
+ public Integer getBorderForAgeForBuildId(int days);
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index 6bdb464..c7b20b7 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -16,7 +16,6 @@
*/
package org.apache.ignite.tcignited;
-import com.google.common.collect.Sets;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
@@ -554,12 +553,17 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return null;
}
- //@GuavaCached(maximumSize = 100000, expireAfterAccessSecs = 90, softValues = true)
+ /** {@inheritDoc} */
public Long getBuildStartTime(int buildId) {
return histCollector.getBuildStartTime(srvIdMaskHigh, buildId);
}
/** {@inheritDoc} */
+ @Override public Integer getBorderForAgeForBuildId(int days) {
+ return buildStartTimeStorage.getBorderForAgeForBuildId(srvIdMaskHigh, days);
+ }
+
+ /** {@inheritDoc} */
@GuavaCached(maximumSize = 500, expireAfterAccessSecs = 30, softValues = true)
@Override public FatBuildCompacted getFatBuild(int buildId, SyncMode mode) {
FatBuildCompacted existingBuild = getFatBuildFromIgnite(buildId);
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
index ab93519..5785a4d 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -18,6 +18,25 @@ package org.apache.ignite.tcignited.history;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiPredicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.cache.Cache;
+import javax.inject.Inject;
+import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.binary.BinaryObject;
@@ -42,20 +61,6 @@ import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.cache.Cache;
-import javax.inject.Inject;
-import javax.inject.Provider;
-import java.time.Duration;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.BiPredicate;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
/**
*
*/
@@ -147,24 +152,19 @@ public class HistoryCollector {
long curTs = System.currentTimeMillis();
Set<Integer> buildIds = bRefsList.stream()
.filter(b -> {
- /* AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId);
- int outOfScopeBuildId = biggestIdOutOfScope == null ? -1 : biggestIdOutOfScope.get();
- return b.id() > outOfScopeBuildId;
- */
-
Integer maxBuildIdForDay = buildStartTimeStorage.getBorderForAgeForBuildId(srvId, TcBotConst.HISTORY_BUILD_ID_BORDER_DAYS);
if (maxBuildIdForDay == null)
return true;
- return b.id()>maxBuildIdForDay;
+ return b.id() > maxBuildIdForDay;
})
.filter(this::applicableForHistory)
.map(BuildRefCompacted::id)
.filter(bId -> !knownBuilds.contains(bId)).collect(Collectors.toSet());
- System.out.println("***** Loading build start time history for suite "
+ logger.info("***** Loading build start time history for suite "
+ compactor.getStringFromId(buildTypeId)
+ " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + buildIds.size() + " builds" );
@@ -207,7 +207,7 @@ public class HistoryCollector {
}
).collect(Collectors.toSet());
- System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " +
+ logger.info("*** Build " + btId + " branch " + branchId + " builds in scope " +
buildInScope.size() + " from " + bRefsList.size());
return buildInScope;