You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/25 18:37:55 UTC
[ignite-teamcity-bot] branch ignite-9848-load-all-builds updated:
Continious reloading bug was fixed,
extended monitoring and tasks naming fixed
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-load-all-builds
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9848-load-all-builds by this push:
new d090e8f Continious reloading bug was fixed, extended monitoring and tasks naming fixed
d090e8f is described below
commit d090e8f2f9510b57f296d4c3dac9d1b89524f325
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Oct 25 21:37:10 2018 +0300
Continious reloading bug was fixed, extended monitoring and tasks naming fixed
---
.../main/java/org/apache/ignite/ci/ITeamcity.java | 2 ++
.../apache/ignite/ci/IgniteTeamcityConnection.java | 1 +
.../org/apache/ignite/ci/di/MonitoredTask.java | 2 ++
.../ignite/ci/di/MonitoredTaskInterceptor.java | 36 ++++++++++++++++------
.../ci/github/ignited/GitHubConnIgnitedImpl.java | 2 +-
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 18 ++++++++---
.../ignite/ci/teamcity/ignited/BuildRefDao.java | 2 ++
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 13 +++++++-
.../ignited/fatbuild/FatBuildCompacted.java | 2 +-
9 files changed, 62 insertions(+), 16 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 17eabb1..115bec0 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -103,9 +103,11 @@ public interface ITeamcity extends ITeamcityConn {
List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId, String branch, Integer sinceBuildId);
/** */
+ @Deprecated
CompletableFuture<List<BuildRef>> getRunningBuilds(@Nullable String branch);
/** */
+ @Deprecated
CompletableFuture<List<BuildRef>> getQueuedBuilds(@Nullable String branch);
/**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
index 5d50df8..12810df 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
@@ -387,6 +387,7 @@ public class IgniteTeamcityConnection implements ITeamcity {
* @param untilDate Until date.
* @param sinceBuildId Since build id. Value is ignored if dates filter is present.
*/
+ @Deprecated
private List<BuildRef> getBuildHistory(@Nullable String buildTypeId,
@Nullable String branchName,
boolean dfltFilter,
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTask.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTask.java
index 268d1cd..d32d8ae 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTask.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTask.java
@@ -32,4 +32,6 @@ public @interface MonitoredTask {
* Argument index (0-based) to be used to extend name.
*/
int nameExtArgIndex() default -1;
+
+ int[] nameExtArgsIndexes() default {};
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTaskInterceptor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTaskInterceptor.java
index 262530c..a0840da 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTaskInterceptor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTaskInterceptor.java
@@ -111,29 +111,47 @@ public class MonitoredTaskInterceptor implements MethodInterceptor {
@NotNull
private String taskName(MethodInvocation invocation) {
- final Method method = invocation.getMethod();
- final String cls = method.getDeclaringClass().getSimpleName();
- final String mtd = method.getName();
+ final Method invocationMtd = invocation.getMethod();
+ final String cls = invocationMtd.getDeclaringClass().getSimpleName();
+ final String mtd = invocationMtd.getName();
- String fullKey ;
+ StringBuilder fullKey= new StringBuilder();
- final MonitoredTask annotation = method.getAnnotation(MonitoredTask.class);
+ final MonitoredTask annotation = invocationMtd.getAnnotation(MonitoredTask.class);
if (annotation != null) {
String activityName = annotation.name();
- fullKey = !Strings.isNullOrEmpty(activityName) ? activityName : cls + "." + mtd;
+ if (!Strings.isNullOrEmpty(activityName))
+ fullKey.append(activityName);
+ else
+ fullKey.append(cls).append(".").append(mtd);
final Object[] arguments = invocation.getArguments();
final int idx = annotation.nameExtArgIndex();
if (arguments != null && idx >= 0 && idx < arguments.length)
- fullKey += "." + Objects.toString(arguments[idx]);
+ fullKey.append(".").append(arguments[idx]);
+
+ int[] ints = annotation.nameExtArgsIndexes();
+
+ for (int i = 0; i < ints.length; i++) {
+ int argIdx = ints[i];
+ if (arguments != null && argIdx >= 0 && argIdx < arguments.length) {
+ if (i == 0)
+ fullKey.append(":");
+ else
+ fullKey.append(", ");
+
+ fullKey.append(arguments[argIdx]);
+ }
+ }
+
}
else
- fullKey = cls + "." + mtd;
+ fullKey.append(cls).append(".").append(mtd);
- return fullKey;
+ return fullKey.toString();
}
public Collection<Invocation> getList() {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
index d75b056..3b998ee 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
@@ -118,7 +118,7 @@ class GitHubConnIgnitedImpl implements IGitHubConnIgnited {
* @param srvId Server id.
* @param fullReindex Reindex all open PRs
*/
- @MonitoredTask(name = "Actualize PRs, full resync", nameExtArgIndex = 1)
+ @MonitoredTask(name = "Actualize PRs(srv, full resync)", nameExtArgsIndexes = {0, 1})
@AutoProfiling
protected String runAtualizePrs(String srvId, boolean fullReindex) {
AtomicReference<String> outLinkNext = new AtomicReference<>();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index 3c94127..4c4219d 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -23,6 +23,7 @@ import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -128,7 +129,7 @@ public class BuildChainProcessor {
contexts.forEach(multiCtx -> {
analyzeTests(multiCtx, teamcity, procLog);
- fillBuildCounts(multiCtx, teamcity, includeScheduledInfo);
+ fillBuildCounts(multiCtx, teamcityIgnited, includeScheduledInfo);
});
if (teamcity != null) {
@@ -263,7 +264,8 @@ public class BuildChainProcessor {
return prevVal == null;
}
- private static void fillBuildCounts(MultBuildRunCtx outCtx, ITeamcity teamcity, boolean includeScheduledInfo) {
+ private static void fillBuildCounts(MultBuildRunCtx outCtx,
+ ITeamcityIgnited teamcityIgnited, boolean includeScheduledInfo) {
if (includeScheduledInfo && !outCtx.hasScheduledBuildsInfo()) {
Function<List<BuildRef>, Long> cntRelatedToThisBuildType = list ->
list.stream()
@@ -271,8 +273,16 @@ public class BuildChainProcessor {
.filter(ref -> Objects.equals(normalizeBranch(outCtx.branchName()), normalizeBranch(ref)))
.count();
- outCtx.setRunningBuildCount(teamcity.getRunningBuilds("").thenApply(cntRelatedToThisBuildType));
- outCtx.setQueuedBuildCount(teamcity.getQueuedBuilds("").thenApply(cntRelatedToThisBuildType));
+ //todo queue all builds instead of specific + caching
+ long cntRunning = teamcityIgnited.getBuildHistory(outCtx.buildTypeId(), outCtx.branchName())
+ .stream().filter(BuildRef::isNotCancelled).filter(BuildRef::isRunning).count();
+
+ outCtx.setRunningBuildCount(CompletableFuture.completedFuture(cntRunning));
+
+ long cntQueued = teamcityIgnited.getBuildHistory(outCtx.buildTypeId(), outCtx.branchName())
+ .stream().filter(BuildRef::isNotCancelled).filter(BuildRef::isQueued).count();
+
+ outCtx.setQueuedBuildCount(CompletableFuture.completedFuture(cntQueued));
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
index 1e38c73..dbea749 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
@@ -32,6 +32,7 @@ import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.ci.db.TcHelperDb;
+import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.util.GridIntList;
@@ -133,6 +134,7 @@ public class BuildRefDao {
/**
* @param srvId Server id.
*/
+ @AutoProfiling
public List<BuildRefCompacted> getQueuedAndRunning(int srvId) {
GridIntList list = new GridIntList(2);
Integer stateQueuedId = compactor.getStringIdIfPresent(BuildRef.STATE_QUEUED);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 14a0e4d..fea9a9d 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -24,6 +24,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.OptionalInt;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -51,6 +52,9 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Logger. */
private static final Logger logger = LoggerFactory.getLogger(TestCompacted.class);
+ /** Max build id diff to enforce reload during incremental refresh. */
+ public static final int MAX_ID_DIFF_TO_ENFORCE = 5000;
+
/** Server id. */
private String srvId;
@@ -164,8 +168,14 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
void actualizeRecentBuilds() {
List<BuildRefCompacted> running = buildRefDao.getQueuedAndRunning(srvIdMaskHigh);
+ //todo intersect with ever found during current launch
Set<Integer> collect = running.stream().map(BuildRefCompacted::id).collect(Collectors.toSet());
+ OptionalInt max = collect.stream().mapToInt(i -> i).max();
+ if (max.isPresent())
+ collect = collect.stream().filter(id -> id > (max.getAsInt() - MAX_ID_DIFF_TO_ENFORCE)).collect(Collectors.toSet());
+ //drop stucked builds.
+
runActualizeBuilds(srvId, false, collect);
// schedule full resync later
@@ -191,7 +201,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
* @param fullReindex Reindex all builds from TC history.
* @param mandatoryToReload Build ID can be used as end of syncing. Ignored if fullReindex mode.
*/
- @MonitoredTask(name = "Actualize BuildRefs, full resync", nameExtArgIndex = 1)
+ @MonitoredTask(name = "Actualize BuildRefs(srv, full resync)", nameExtArgsIndexes = {0, 1})
@AutoProfiling
protected String runActualizeBuilds(String srvId, boolean fullReindex,
@Nullable Set<Integer> mandatoryToReload) {
@@ -206,6 +216,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
final Set<Integer> stillNeedToFind =
mandatoryToReload == null ? Collections.emptySet() : Sets.newHashSet(mandatoryToReload);
+ tcDataFirstPage.stream().map(BuildRef::getId).forEach(stillNeedToFind::remove);
while (outLinkNext.get() != null) {
String nextPageUrl = outLinkNext.get();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 25e6df5..29ec877 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -48,7 +48,7 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
public static final int COMPOSITE_F = 2;
/** Entity fields version. */
- private short _ver;
+ private short _ver = LATEST_VERSION;
/** Start date. The number of milliseconds since January 1, 1970, 00:00:00 GMT */
private long startDate;