You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/11/30 13:21:43 UTC
[ignite-teamcity-bot] branch master updated: IGNITE-9542 Fix
performance issue with tests counting
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 868cfa7 IGNITE-9542 Fix performance issue with tests counting
868cfa7 is described below
commit 868cfa7ee5a68574a5fb2169d952030eb17207ad
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Nov 30 16:21:11 2018 +0300
IGNITE-9542 Fix performance issue with tests counting
---
.../ci/teamcity/ignited/runhist/RunHistSync.java | 72 +++++++++++++---------
1 file changed, 43 insertions(+), 29 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
index 11d43c4..a74217c 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
@@ -103,46 +103,49 @@ public class RunHistSync {
int branchNameNormalized = compactor.getStringId(normalizeBranch(build.branchName(compactor)));
+ AtomicInteger cntTests = new AtomicInteger();
Map<RunHistKey, List<Invocation>> testInvMap = new HashMap<>();
build.getAllTests().forEach(t -> {
RunHistKey histKey = new RunHistKey(srvId, t.testName(), branchNameNormalized);
List<Invocation> list = testInvMap.computeIfAbsent(histKey, k -> new ArrayList<>());
- Invocation inv = t.toInvocation(compactor, build);
- list.add(inv);
+ list.add(t.toInvocation(compactor, build));
+
+ cntTests.incrementAndGet();
});
RunHistKey buildInvKey = new RunHistKey(srvId, build.buildTypeId(), branchNameNormalized);
Invocation buildInv = build.toInvocation(compactor);
- int cnt = containedTestsCnt(testInvMap);
+ int cnt = cntTests.get();
synchronized (this) {
final SyncTask syncTask = buildToSave.computeIfAbsent(srvVame, s -> new SyncTask());
- if (syncTask.sheduled() + cnt <= MAX_TESTS_QUEUE)
- syncTask.addLater(testInvMap, buildInvKey, buildInv);
+ if (syncTask.sheduledTestsCnt() + cnt <= MAX_TESTS_QUEUE)
+ syncTask.addLater(testInvMap, cnt, buildInvKey, buildInv);
else
saveNow = true;
}
if (saveNow) {
- Map<RunHistKey, List<Invocation>> suiteAsMap =
+ saveInvocationsMap(
Collections.singletonMap(buildInvKey,
- Collections.singletonList(buildInv));
- saveInvocationsMap(suiteAsMap, testInvMap);
+ Collections.singletonList(buildInv)
+ ),
+ testInvMap);
}
else {
int ldrToActivate = ThreadLocalRandom.current().nextInt(HIST_LDR_TASKS) + 1;
scheduler.sheduleNamed(taskName("saveBuildToHistory." + ldrToActivate, srvVame),
- () -> saveBuildToHistory(srvVame, ldrToActivate), 2, TimeUnit.MINUTES);
+ () -> saveBuildToHistory(srvVame, ldrToActivate), 1, TimeUnit.MINUTES);
}
}
@MonitoredTask(name="Save Builds To History(srv, runner)", nameExtArgsIndexes = {0,1})
@SuppressWarnings("WeakerAccess")
protected String saveBuildToHistory(String srvName, int ldrToActivate) {
- Map<RunHistKey, List<Invocation>> saveThisRun;
+ Map<RunHistKey, List<Invocation>> testsSaveThisRun;
Map<RunHistKey, List<Invocation>> buildsSaveThisRun;
synchronized (this) {
@@ -150,14 +153,14 @@ public class RunHistSync {
if (syncTask == null)
return "Nothing to sync";
- saveThisRun = syncTask.tests;
- buildsSaveThisRun = syncTask.suites;
-
- syncTask.tests = new HashMap<>();
- syncTask.suites = new HashMap<>();
+ buildsSaveThisRun = syncTask.takeSuites();
+ testsSaveThisRun = syncTask.takeTests();
}
- return saveInvocationsMap(buildsSaveThisRun, saveThisRun);
+ if(buildsSaveThisRun.isEmpty() && testsSaveThisRun.isEmpty())
+ return "Nothing to sync";
+
+ return saveInvocationsMap(buildsSaveThisRun, testsSaveThisRun);
}
@AutoProfiling
@@ -241,7 +244,7 @@ public class RunHistSync {
public void invokeLaterFindMissingHistory(String srvName) {
scheduler.sheduleNamed(taskName("findMissingHistFromBuildRef", srvName),
- () -> findMissingHistFromBuildRef(srvName), 360, TimeUnit.MINUTES);
+ () -> findMissingHistFromBuildRef(srvName), 12, TimeUnit.HOURS);
}
@NotNull
@@ -309,28 +312,39 @@ public class RunHistSync {
* Scope of work: builds to be loaded from a connection.
*/
private static class SyncTask {
- Map<RunHistKey, List<Invocation>> suites = new HashMap<>();
- Map<RunHistKey, List<Invocation>> tests = new HashMap<>();
+ private Map<RunHistKey, List<Invocation>> suites = new HashMap<>();
+ private AtomicInteger testCnt = new AtomicInteger();
+ private Map<RunHistKey, List<Invocation>> tests = new HashMap<>();
- public int sheduled() {
- return containedTestsCnt(this.tests);
+ public int sheduledTestsCnt() {
+ return testCnt.get();
}
public void addLater(Map<RunHistKey, List<Invocation>> testInvMap,
- RunHistKey buildInvKey,
+ int testCnt, RunHistKey buildInvKey,
Invocation buildInv) {
suites
.computeIfAbsent(buildInvKey, k -> new ArrayList<>())
.add(buildInv);
tests.putAll(testInvMap);
+ this.testCnt.addAndGet(testCnt);
}
- }
- /**
- * @param tests Tests.
- * @return count of invocations.
- */
- public static int containedTestsCnt(Map<RunHistKey, List<Invocation>> tests) {
- return tests.values().stream().mapToInt(List::size).sum();
+ private Map<RunHistKey, List<Invocation>> takeTests() {
+ Map<RunHistKey, List<Invocation>> saveThisRun = tests;
+
+ tests = new HashMap<>();
+ testCnt.set(0);
+
+ return saveThisRun;
+ }
+
+ private Map<RunHistKey, List<Invocation>> takeSuites() {
+ Map<RunHistKey, List<Invocation>> buildsSaveThisRun = suites;
+
+ suites = new HashMap<>();
+
+ return buildsSaveThisRun;
+ }
}
}