You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/06/21 15:28:32 UTC
[ignite-teamcity-bot] branch test-hist-performance updated: Trusted
tests & suite history performance fixes: border implemented for skipping
too old builds
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/test-hist-performance by this push:
new 0895500 Trusted tests & suite history performance fixes: border implemented for skipping too old builds
0895500 is described below
commit 08955001698a34ff8e1a44791c2d01727d09c962
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Jun 21 18:28:18 2019 +0300
Trusted tests & suite history performance fixes: border implemented for skipping too old builds
---
build.gradle | 5 ++--
ignite-tc-helper-web/build.gradle | 4 ---
jetty-launcher/build.gradle | 4 ---
tcbot-server-node/build.gradle | 3 --
.../ignite/tcignited/history/HistoryCollector.java | 33 +++++++++++++++++++---
.../ignite/tcignited/history/SuiteInvocation.java | 2 ++
.../history/SuiteInvocationHistoryDao.java | 5 ++--
7 files changed, 37 insertions(+), 19 deletions(-)
diff --git a/build.gradle b/build.gradle
index edf7b84..43f7f5a 100644
--- a/build.gradle
+++ b/build.gradle
@@ -38,10 +38,11 @@ subprojects {
}
allprojects {
+ apply plugin: 'java'
+ sourceCompatibility = '1.8'
+ targetCompatibility = '1.8'
ext {
- sourceCompatibility = '1.8'
- targetCompatibility = '1.8'
jettyVer = '9.4.12.v20180830'
diff --git a/ignite-tc-helper-web/build.gradle b/ignite-tc-helper-web/build.gradle
index a9cb41f..e4fa011 100644
--- a/ignite-tc-helper-web/build.gradle
+++ b/ignite-tc-helper-web/build.gradle
@@ -18,10 +18,6 @@
apply plugin: 'java'
apply plugin: 'war'
-sourceCompatibility = '1.8'
-targetCompatibility = '1.8'
-
-
// https://www.apache.org/legal/resolved.html#category-a
dependencies {
compile (project(":tcbot-common"));
diff --git a/jetty-launcher/build.gradle b/jetty-launcher/build.gradle
index edfb84d..5df3d7b 100644
--- a/jetty-launcher/build.gradle
+++ b/jetty-launcher/build.gradle
@@ -18,10 +18,6 @@
apply plugin: 'java'
apply plugin: 'application'
-sourceCompatibility = '1.8'
-targetCompatibility = '1.8'
-
-
mainClassName = 'org.apache.ignite.ci.TcHelperJettyLauncher'
applicationDefaultJvmArgs = ["-Dteamcity.helper.home=../work",
"-Dteamcity.bot.regionsize=16", // 16g Durable Memory region
diff --git a/tcbot-server-node/build.gradle b/tcbot-server-node/build.gradle
index 3ffcf74..0bc43f8 100644
--- a/tcbot-server-node/build.gradle
+++ b/tcbot-server-node/build.gradle
@@ -18,9 +18,6 @@
apply plugin: 'java'
apply plugin: 'application'
-sourceCompatibility = '1.8'
-targetCompatibility = '1.8'
-
repositories {
mavenCentral()
mavenLocal()
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
index 84c4f7d..b204eae 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -24,8 +24,11 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.inject.Inject;
@@ -105,6 +108,7 @@ public class HistoryCollector {
return hist.getTestRunHist(testName);
}
+ ConcurrentMap<Integer, AtomicInteger> biggestBuildIdOutOfHistoryScope = new ConcurrentHashMap<>();
/**
* Latest actual Build ids supplier. This supplier should handle all equivalent branches in
@@ -121,11 +125,16 @@ public class HistoryCollector {
String branchId = compactor.getStringFromId(normalizedBaseBranch);
List<BuildRefCompacted> compacted = buildRefDao.getAllBuildsCompacted(srvId, btId,
branchEquivalence.branchForQuery(branchId));
+
+ AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId);
+ int outOfScopeBuildId = biggestIdOutOfScope == null ? -1 : biggestIdOutOfScope.get();
+
long curTs = System.currentTimeMillis();
Set<Integer> buildIds = compacted.stream()
- .filter(b -> !knownBuilds.contains(b.id()))
+ .filter(b -> b.id() > outOfScopeBuildId)
.filter(this::applicableForHistory)
- .map(BuildRefCompacted::id).collect(Collectors.toSet());
+ .map(BuildRefCompacted::id)
+ .filter(bId -> !knownBuilds.contains(bId)).collect(Collectors.toSet());
System.out.println("***** Loading build start time history for suite "
+ compactor.getStringFromId(buildTypeId)
@@ -153,7 +162,23 @@ public class HistoryCollector {
if (startTime == null)
return false;
- return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS;
+ long ageInDays = Duration.ofMillis(curTs - startTime).toDays();
+
+ if(ageInDays>InvocationData.MAX_DAYS + 2) {
+ AtomicInteger integer = biggestBuildIdOutOfHistoryScope.computeIfAbsent(srvId,
+ s -> {
+ AtomicInteger atomicInteger = new AtomicInteger();
+ atomicInteger.set(-1);
+ return atomicInteger;
+ });
+
+ int newBorder = integer.accumulateAndGet(bId, Math::max);
+
+ if (newBorder == bId)
+ System.err.println(" New border for server was set " + bId);
+ }
+
+ return ageInDays < InvocationData.MAX_DAYS;
}
).collect(Collectors.toSet());
@@ -258,7 +283,7 @@ public class HistoryCollector {
+ " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": added " +
suiteRunHist.size() + " invocations from " + missedBuildsIds.size() + " builds checked");
- histDao.putAll(srvId, suiteRunHist);
+ histDao.putAllAsync(srvId, suiteRunHist);
return suiteRunHist;
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
index 0687a13..8bc8d95 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocation.java
@@ -20,6 +20,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.function.BiPredicate;
+import org.apache.ignite.cache.affinity.AffinityKeyMapped;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
@@ -37,6 +38,7 @@ public class SuiteInvocation {
private int srvId;
/** Suite name for queries */
+ @AffinityKeyMapped
@QuerySqlField(orderedGroups = {@QuerySqlField.Group(name = "serverSuiteBranch", order = 1)})
private int buildTypeId;
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
index f252a85..b8504fa 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
@@ -76,11 +76,12 @@ public class SuiteInvocationHistoryDao {
return map;
}
- public void putAll(int srvId, Map<Integer, SuiteInvocation> addl) {
+ @AutoProfiling
+ public void putAllAsync(int srvId, Map<Integer, SuiteInvocation> addl) {
Map<Long, SuiteInvocation> data = new HashMap<>();
addl.forEach((k, v) -> data.put(BuildRefDao.buildIdToCacheKey(srvId, k), v));
- suiteHistory.putAll(data);
+ suiteHistory.putAllAsync(data);
}
}