You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/06/20 16:37:17 UTC
[ignite-teamcity-bot] branch test-hist-performance updated: Trusted
tests & suite history performance fixes:
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/test-hist-performance by this push:
new a8aa63d Trusted tests & suite history performance fixes:
a8aa63d is described below
commit a8aa63df3683975373110276fbcc00f33971773a
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Jun 20 19:36:58 2019 +0300
Trusted tests & suite history performance fixes:
---
.../apache/ignite/tcignited/build/FatBuildDao.java | 35 ++++++++++++------
.../ignite/tcignited/history/HistoryCollector.java | 41 +++++++++++++++++-----
.../tcignited/history/RunHistCompactedDao.java | 35 ++++++++++++++++++
.../history/SuiteInvocationHistoryDao.java | 2 ++
4 files changed, 93 insertions(+), 20 deletions(-)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index 98ef8fd..36f355c 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -18,6 +18,7 @@
package org.apache.ignite.tcignited.build;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@@ -37,6 +38,7 @@ import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.cache.CacheEntryProcessor;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
@@ -202,31 +204,42 @@ public class FatBuildDao {
.filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvId, id)).collect(Collectors.toSet());
}
+ /**
+ * @param srvId Server id.
+ * @param ids Ids.
+ */
public Map<Integer, Long> getBuildStartTime(int srvId, Set<Integer> ids) {
- Map<Long, EntryProcessorResult<Long>> map = buildsCache.invokeAll(buildsIdsToCacheKeys(srvId, ids), new GetStartTimeProc());
-
+ IgniteCache<Long, BinaryObject> cacheBin = buildsCache.withKeepBinary();
+ Set<Long> keys = buildsIdsToCacheKeys(srvId, ids);
HashMap<Integer, Long> res = new HashMap<>();
- map.forEach((k, r)->{
- Long aLong = r.get();
- if(aLong!=null)
- res.put(BuildRefDao.cacheKeyToBuildId(k), aLong);
- });
+ Iterables.partition(keys, 32 * 10).forEach(
+ chunk -> {
+ Map<Long, EntryProcessorResult<Long>> map = cacheBin.invokeAll(keys, new GetStartTimeProc());
+ map.forEach((k, r) -> {
+ Long ts = r.get();
+ if (ts != null)
+ res.put(BuildRefDao.cacheKeyToBuildId(k), ts);
+ });
+ }
+ );
return res;
}
- private static class GetStartTimeProc implements CacheEntryProcessor<Long, FatBuildCompacted, Long> {
+ private static class GetStartTimeProc implements CacheEntryProcessor<Long, BinaryObject, Long> {
public GetStartTimeProc() {
}
- @Override public Long process(MutableEntry<Long, FatBuildCompacted> entry,
+ /** {@inheritDoc} */
+ @Override public Long process(MutableEntry<Long, BinaryObject> entry,
Object... arguments) throws EntryProcessorException {
if (entry.getValue() == null)
return null;
- FatBuildCompacted fatBuildCompacted = entry.getValue();
- return fatBuildCompacted.getStartDateTs();
+ BinaryObject buildBinary = entry.getValue();
+
+ return buildBinary.field("startDate");
}
}
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
index f5b720e..f16a1ca 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -126,14 +126,28 @@ public class HistoryCollector {
.filter(this::applicableForHistory)
.map(BuildRefCompacted::id).collect(Collectors.toSet());
- Map<Integer, Long> buildStartTimeFromFatBuild = getStartTimeFromFatBuild(srvId, buildIds);
+ System.out.println("***** Loading build start time history for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + buildIds.size() + " builds" );
+
+ Map<Integer, Long> buildStartTimeFromSeparatedCache = getStartTimeFromSpecialCache(srvId, buildIds);
+
+ HashSet<Integer> notFoundKeys = new HashSet<>(buildIds);
+ notFoundKeys.removeAll(buildStartTimeFromSeparatedCache.keySet());
+
+ if (!notFoundKeys.isEmpty()) {
+ Map<Integer, Long> buildStartTimeFromFatBuild = getStartTimeFromFatBuild(srvId, notFoundKeys);
+ buildStartTimeFromSeparatedCache.putAll(buildStartTimeFromFatBuild);
+
+ runHistCompactedDao.setBuildsStartTime(srvId, buildStartTimeFromFatBuild);
+ }
//todo filter queued, cancelled and so on
Set<Integer> buildInScope = buildIds.stream().filter(
bId -> {
//todo getAll
//todo getStartTime From FatBuild
- Long startTime = buildStartTimeFromFatBuild.get(bId);
+ Long startTime = buildStartTimeFromSeparatedCache.get(bId);
//todo cache in runHistCompactedDao.getBuildStartTime(srvId, bRef.id());
if (startTime == null)
return false;
@@ -143,12 +157,17 @@ public class HistoryCollector {
).collect(Collectors.toSet());
System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " +
- buildInScope+ " from " + buildIds.size());
+ buildInScope.size() + " from " + buildIds.size());
return buildIds;
}
@AutoProfiling
+ protected Map<Integer, Long> getStartTimeFromSpecialCache(int srvId, Set<Integer> buildIds) {
+ return runHistCompactedDao.getBuildsStartTime(srvId, buildIds);
+ }
+
+ @AutoProfiling
protected Map<Integer, Long> getStartTimeFromFatBuild(int srvId, Set<Integer> buildIds) {
return fatBuildDao.getBuildStartTime(srvId, buildIds);
}
@@ -166,6 +185,10 @@ public class HistoryCollector {
int normalizedBaseBranch) {
Map<Integer, SuiteInvocation> suiteRunHist = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
+ System.out.println("***** Found history for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + suiteRunHist.size() );
+
Set<Integer> buildIds = determineLatestBuilds(srvId, buildTypeId, normalizedBaseBranch, suiteRunHist.keySet());
HashSet<Integer> missedBuildsIds = new HashSet<>(buildIds);
@@ -175,12 +198,6 @@ public class HistoryCollector {
if (!missedBuildsIds.isEmpty()) {
Map<Integer, SuiteInvocation> addl = addSuiteInvocationsToHistory(srvId, missedBuildsIds, normalizedBaseBranch);
- System.err.println("***** + Adding to persisted history for suite "
- + compactor.getStringFromId(buildTypeId)
- + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": added " +
- addl.size() + " invocations from " + missedBuildsIds + " builds checked");
-
- histDao.putAll(srvId, addl);
suiteRunHist.putAll(addl);
}
@@ -230,6 +247,12 @@ public class HistoryCollector {
suiteRunHist.put(fatBuildCompacted.id(), sinv);
});
+ System.err.println("***** + Adding to persisted history "
+ + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": added " +
+ suiteRunHist.size() + " invocations from " + missedBuildsIds.size() + " builds checked");
+
+ histDao.putAll(srvId, suiteRunHist);
+
return suiteRunHist;
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
index e6478a7..ffc0c47 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
@@ -17,9 +17,15 @@
package org.apache.ignite.tcignited.history;
+import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.cache.Cache;
@@ -40,6 +46,7 @@ import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.GuavaCached;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
@@ -235,4 +242,32 @@ public class RunHistCompactedDao {
cluster.disableWal(testHistCache.getName());
cluster.disableWal(suiteHistCache.getName());
}
+
+ private static Set<Long> buildsIdsToCacheKeys(int srvId, Collection<Integer> ids) {
+ return ids.stream()
+ .filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvId, id)).collect(Collectors.toSet());
+ }
+
+ public Map<Integer, Long> getBuildsStartTime(int srvId, Set<Integer> ids) {
+ Set<Long> cacheKeys = buildsIdsToCacheKeys(srvId, ids);
+
+ Map<Integer, Long> res = new HashMap<>();
+
+ buildStartTime.getAll(cacheKeys).forEach((k, r) -> {
+ res.put(BuildRefDao.cacheKeyToBuildId(k), r);
+ });
+
+ return res;
+ }
+
+ public void setBuildsStartTime(int srvId, Map<Integer, Long> builds) {
+ Map<Long, Long> res = new HashMap<>();
+
+ builds.forEach((buildId, ts) -> {
+ if (ts != null)
+ res.put(buildIdToCacheKey(srvId, buildId), ts);
+ });
+
+ buildStartTime.putAll(res);
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
index 3b5b450..f252a85 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteInvocationHistoryDao.java
@@ -30,6 +30,7 @@ import org.apache.ignite.cache.QueryEntity;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.SqlQuery;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
@@ -58,6 +59,7 @@ public class SuiteInvocationHistoryDao {
suiteHistory = ignite.getOrCreateCache(ccfg);
}
+ @AutoProfiling
public Map<Integer, SuiteInvocation> getSuiteRunHist(int srvId, int buildTypeId, int normalizedBranchName) {
java.util.Map<Integer, SuiteInvocation> map = new HashMap<>();
try (QueryCursor<Cache.Entry<Long, SuiteInvocation>> qryCursor = suiteHistory.query(