You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/06/20 14:22:29 UTC
[ignite-teamcity-bot] 06/07: Trusted tests & suite history
performance fixes: history collection was moved to standalone class
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch test-hist-performance
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 685ad52962f4b3c23590433bda7ea52f4e2a7726
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Jun 20 15:22:11 2019 +0300
Trusted tests & suite history performance fixes: history collection was moved to standalone class
---
.../ignite/tcignited/TeamcityIgnitedImpl.java | 58 ++---
.../ignite/tcignited/TeamcityIgnitedModule.java | 2 +
.../apache/ignite/tcignited/build/FatBuildDao.java | 213 +--------------
.../ignite/tcignited/build/SuiteHistory.java | 14 +
.../BranchEquivalence.java} | 35 ++-
.../ignite/tcignited/buildref/BuildRefDao.java | 2 +-
.../ignite/tcignited/history/HistoryCollector.java | 290 +++++++++++++++++++++
7 files changed, 346 insertions(+), 268 deletions(-)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index dfdf970..18ea52b 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -61,12 +61,15 @@ import org.apache.ignite.tcbot.persistence.scheduler.IScheduler;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
import org.apache.ignite.tcignited.buildlog.BuildLogCheckResultDao;
+import org.apache.ignite.tcignited.buildref.BranchEquivalence;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.tcignited.buildref.BuildRefSync;
+import org.apache.ignite.tcignited.history.HistoryCollector;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.IRunStat;
import org.apache.ignite.tcignited.history.RunHistCompactedDao;
import org.apache.ignite.tcignited.history.RunHistSync;
+import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
import org.apache.ignite.tcignited.mute.MuteDao;
import org.apache.ignite.tcignited.mute.MuteSync;
import org.apache.ignite.tcservice.ITeamcity;
@@ -94,11 +97,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Max build id diff to enforce reload during incremental refresh. */
public static final int MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN = 3000;
- /** Default synonyms. */
- private static final List<String> DEFAULT_SYNONYMS
- = Collections.unmodifiableList(
- Lists.newArrayList(ITeamcity.DEFAULT, ITeamcity.REFS_HEADS_MASTER, ITeamcity.MASTER));
-
/** Server (service) code. */
private String srvCode;
@@ -150,11 +148,20 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Run history sync. */
@Inject private RunHistSync runHistSync;
- @Inject private BuildLogCheckResultDao logCheckResultDao;
+ /** Logger check result DAO. */
+ @Inject private BuildLogCheckResultDao logCheckResDao;
+
+ /** History DAO. */
+ @Inject private SuiteInvocationHistoryDao histDao;
+
+ /** History collector. */
+ @Inject private HistoryCollector histCollector;
/** Strings compactor. */
@Inject private IStringCompactor compactor;
+ @Inject private BranchEquivalence branchEquivalence;
+
/** Server ID mask for cache Entries. */
private int srvIdMaskHigh;
@@ -171,7 +178,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
changesDao.init();
runHistCompactedDao.init();
muteDao.init();
- logCheckResultDao.init();
+ logCheckResDao.init();
+ histDao.init();
}
/**
@@ -351,7 +359,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
@Nullable String branchName) {
ensureActualizeRequested();
- return buildRefDao.getAllBuildsCompacted(srvIdMaskHigh, buildTypeId, branchForQuery(branchName));
+ return buildRefDao.getAllBuildsCompacted(srvIdMaskHigh, buildTypeId, branchEquivalence.branchForQuery(branchName));
}
/** {@inheritDoc} */
@@ -364,7 +372,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
if (stateQueuedId == null)
return Collections.emptyList();
- Set<Integer> branchNameIds = branchForQuery(branchName).stream().map(str -> compactor.getStringIdIfPresent(str))
+ Set<Integer> branchNameIds = branchEquivalence.branchForQuery(branchName).stream().map(str -> compactor.getStringIdIfPresent(str))
.filter(Objects::nonNull).collect(Collectors.toSet());
List<BuildRefCompacted> res = new ArrayList<>();
@@ -441,30 +449,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
if (testName < 0 || buildTypeId < 0 || normalizedBaseBranch < 0)
return null;
- Function<Set<Integer>, Set<Integer>> supplier = (knownBuilds) -> {
- String btId = compactor.getStringFromId(buildTypeId);
- String branchId = compactor.getStringFromId(normalizedBaseBranch);
- List<BuildRefCompacted> compacted = getAllBuildsCompacted(btId, branchId);
- long curTs = System.currentTimeMillis();
- Set<Integer> buildIds = compacted.stream()
- .filter(b -> !knownBuilds.contains(b.id()))
- //todo filter queued, cancelled and so on
- .filter(
- bRef -> {
- Long startTime = getBuildStartTime(bRef.id());
- if (startTime == null)
- return false;
-
- return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS;
- }
- ).map(BuildRefCompacted::id).collect(Collectors.toSet());
-
- System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size());
-
- return buildIds;
- };
-
- return fatBuildDao.getTestRunHist(srvIdMaskHigh, supplier, testName, buildTypeId, normalizedBaseBranch);
+ return histCollector.getTestRunHist(srvIdMaskHigh, testName, buildTypeId, normalizedBaseBranch);
}
/** {@inheritDoc} */
@@ -509,13 +494,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return buildTypeDao.getFatBuildType(srvIdMaskHigh, buildTypeId);
}
- public List<String> branchForQuery(@Nullable String branchName) {
- if (ITeamcity.DEFAULT.equals(branchName))
- return DEFAULT_SYNONYMS;
- else
- return Collections.singletonList(branchName);
- }
-
/**
* Enables scheduling for build refs/builds/history sync
*/
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
index 86cb573..bc0b203 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedModule.java
@@ -29,6 +29,7 @@ import org.apache.ignite.ci.teamcity.ignited.change.ChangeDao;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
+import org.apache.ignite.tcignited.history.HistoryCollector;
import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
import org.apache.ignite.tcignited.mute.MuteDao;
import org.apache.ignite.tcignited.mute.MuteSync;
@@ -67,6 +68,7 @@ public class TeamcityIgnitedModule extends AbstractModule {
bind(MuteSync.class).in(new SingletonScope());
bind(BuildLogCheckResultDao.class).in(new SingletonScope());
bind(SuiteInvocationHistoryDao.class).in(new SingletonScope());
+ bind(HistoryCollector.class).in(new SingletonScope());
TcRealConnectionModule module = new TcRealConnectionModule();
if (conn != null)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index e0dce95..4424158 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -18,49 +18,30 @@
package org.apache.ignite.tcignited.build;
import com.google.common.base.Preconditions;
-import com.google.common.cache.CacheBuilder;
import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.cache.Cache;
-import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.EntryProcessorResult;
-import javax.cache.processor.MutableEntry;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheEntryProcessor;
-import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
-import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
-import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
-import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
-import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
-import org.apache.ignite.tcignited.history.IRunHistory;
-import org.apache.ignite.tcignited.history.SuiteInvocation;
-import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
+import org.apache.ignite.tcignited.history.HistoryCollector;
import org.apache.ignite.tcservice.model.changes.ChangesList;
import org.apache.ignite.tcservice.model.result.Build;
import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence;
import org.apache.ignite.tcservice.model.result.stat.Statistics;
-import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrencesFull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -84,19 +65,8 @@ public class FatBuildDao {
/** Compactor. */
@Inject private IStringCompactor compactor;
- @Inject SuiteInvocationHistoryDao historyDao;
-
- /**
- * Non persistence cache for all suite RunHistory for particular branch.
- * RunHistKey(ServerId||BranchId||suiteId)-> Build reference
- */
- private final com.google.common.cache.Cache<RunHistKey, SuiteHistory> runHistInMemCache
- = CacheBuilder.newBuilder()
- .maximumSize(8000)
- .expireAfterAccess(16, TimeUnit.MINUTES)
- .softValues()
- .build();
-
+ /** History collector. */
+ @Inject private HistoryCollector histCollector;
/**
*
@@ -104,8 +74,6 @@ public class FatBuildDao {
public FatBuildDao init() {
buildsCache = igniteProvider.get().getOrCreateCache(CacheConfigs.getCacheV2Config(TEAMCITY_FAT_BUILD_CACHE_NAME));
- historyDao.init();
-
return this;
}
@@ -158,7 +126,7 @@ public class FatBuildDao {
public void putFatBuild(int srvIdMaskHigh, int buildId, FatBuildCompacted newBuild) {
buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
- invalidateHistoryInMem(srvIdMaskHigh, Stream.of(newBuild));
+ histCollector.invalidateHistoryInMem(srvIdMaskHigh, Stream.of(newBuild));
}
public static int[] extractChangeIds(@Nonnull ChangesList changesList) {
@@ -223,185 +191,12 @@ public class FatBuildDao {
.filter(entry -> isKeyForServer(entry.getKey(), srvId));
}
- /**
- * @param srvIdMaskHigh Server id mask to be placed at high bits in the key.
- * @param buildIdsSupplier Latest actual Build ids supplier. This supplier should handle all equivalent branches in it.
- * @param testName Test name.
- * @param buildTypeId Suite (Build type) id.
- * @param normalizedBaseBranch Branch name.
- */
- @AutoProfiling
- public IRunHistory getTestRunHist(int srvIdMaskHigh,
- Function<Set<Integer>, Set<Integer>> buildIdsSupplier, int testName, int buildTypeId, int normalizedBaseBranch) {
-
- RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
-
- SuiteHistory hist;
- try {
- hist = runHistInMemCache.get(runHistKey,
- () -> loadSuiteHistory(srvIdMaskHigh, buildIdsSupplier, buildTypeId, normalizedBaseBranch));
- }
- catch (ExecutionException e) {
- throw ExceptionUtil.propagateException(e);
- }
-
- return hist.testsHistory.get(testName);
- }
-
-
- //todo create standalone history collector class
- @AutoProfiling
- public SuiteHistory loadSuiteHistory(int srvId,
- Function<Set<Integer>, Set<Integer>> buildIdsSupplier,
- int buildTypeId,
- int normalizedBaseBranch) {
- Map<Integer, SuiteInvocation> suiteRunHist = historyDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
-
- Set<Integer> buildIds = determineLatestBuildsFunction(buildIdsSupplier, suiteRunHist.keySet());
-
- HashSet<Integer> missedBuildsIds = new HashSet<>(buildIds);
-
- missedBuildsIds.removeAll(suiteRunHist.keySet());
-
- if (!missedBuildsIds.isEmpty()) {
- Map<Integer, SuiteInvocation> addl = addSuiteInvocationsToHistory(srvId, missedBuildsIds, normalizedBaseBranch);
-
- System.err.println("***** + Adding to persisted history for suite "
- + compactor.getStringFromId(buildTypeId)
- + " branch " + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
- addl.size() + " invocations");
-
- historyDao.putAll(srvId, addl);
- suiteRunHist.putAll(addl);
- }
-
- SuiteHistory sumary = new SuiteHistory();
-
- suiteRunHist.forEach((buildId, suiteInv) -> {
- suiteInv.tests().forEach((tName, test) -> {
- sumary.testsHistory.computeIfAbsent(tName,
- k_ -> new RunHistCompacted()).innerAddInvocation(test);
- });
-
- });
-
- System.err.println("***** History for suite "
- + compactor.getStringFromId(buildTypeId)
- + " branch" + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
- sumary.size(igniteProvider.get()) + " bytes");
-
- return sumary;
- }
-
- @AutoProfiling
- public Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId,
- HashSet<Integer> missedBuildsIds, int normalizedBaseBranch) {
- Map<Integer, SuiteInvocation> suiteRunHist = new HashMap<>();
- int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
-
- getAllFatBuilds(srvId, missedBuildsIds).forEach((buildCacheKey, fatBuildCompacted) -> {
- SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, (k, v) -> false);
-
- Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
- tests.forEach(
- testCompacted -> {
- Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
-
- sinv.addTest(testCompacted.testName(), invocation);
- }
- );
-
- suiteRunHist.put(fatBuildCompacted.id(), sinv);
- });
-
- return suiteRunHist;
- }
-
- @AutoProfiling
- protected SuiteHistory calcSuiteHistory(int srvIdMaskHigh, Set<Integer> buildIds) {
- Set<Long> cacheKeys = buildsIdsToCacheKeys(srvIdMaskHigh, buildIds);
-
- int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
-
- CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> processor = new HistoryCollectProcessor(successStatusStrId);
-
- Map<Long, EntryProcessorResult<Map<Integer, Invocation>>> map = buildsCache.invokeAll(cacheKeys, processor);
-
- SuiteHistory hist = new SuiteHistory();
-
- map.values().forEach(
- res -> {
- if (res == null)
- return;
-
- Map<Integer, Invocation> invocationMap = res.get();
-
- if (invocationMap == null)
- return;
-
- invocationMap.forEach((k, v) -> {
- RunHistCompacted compacted = hist.testsHistory.computeIfAbsent(k,
- k_ -> new RunHistCompacted());
-
- compacted.innerAddInvocation(v);
- });
-
- }
- );
- System.err.println("Suite history: tests in scope "
- + hist.testsHistory.size()
- + " for " +buildIds.size() + " builds checked"
- + " size " + hist.size(igniteProvider.get()));
-
- return hist;
- }
private static Set<Long> buildsIdsToCacheKeys(int srvIdMaskHigh, Collection<Integer> stream) {
return stream.stream()
.filter(Objects::nonNull).map(id -> buildIdToCacheKey(srvIdMaskHigh, id)).collect(Collectors.toSet());
}
- @AutoProfiling
- protected Set<Integer> determineLatestBuildsFunction(Function<Set<Integer>, Set<Integer>> buildIdsSupplier,
- Set<Integer> known) {
- return buildIdsSupplier.apply(known);
- }
-
- public void invalidateHistoryInMem(int srvId, Stream<BuildRefCompacted> stream) {
- Iterable<RunHistKey> objects =
- stream
- .map(b -> new RunHistKey(srvId, b.buildTypeId(), b.branchName()))
- .collect(Collectors.toSet());
-
- runHistInMemCache.invalidateAll(objects);
- }
-
- private static class HistoryCollectProcessor implements CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> {
- private final int successStatusStrId;
-
- public HistoryCollectProcessor(int successStatusStrId) {
- this.successStatusStrId = successStatusStrId;
- }
-
- @Override public Map<Integer, Invocation> process(MutableEntry<Long, FatBuildCompacted> entry,
- Object... arguments) throws EntryProcessorException {
- if (entry.getValue() == null)
- return null;
-
- Map<Integer, Invocation> hist = new HashMap<>();
- FatBuildCompacted fatBuildCompacted = entry.getValue();
- Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
- tests.forEach(
- testCompacted -> {
- Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
-
- hist.put(testCompacted.testName(), invocation);
- }
- );
-
- return hist;
- }
- }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
index a7af3f8..394b75d 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
@@ -22,8 +22,10 @@ import java.util.Map;
import java.util.TreeMap;
import org.apache.ignite.Ignite;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
import org.apache.ignite.internal.binary.BinaryObjectExImpl;
+import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.SuiteInvocation;
/**
@@ -37,4 +39,16 @@ public class SuiteHistory {
BinaryObjectExImpl binary = ignite.binary().toBinary(this);
return binary.length();
}
+
+ public IRunHistory getTestRunHist(int name) {
+ return testsHistory.get(name);
+ }
+
+ public RunHistCompacted getOrAddTestsHistory(Integer tName) {
+ return testsHistory.computeIfAbsent(tName, k_ -> new RunHistCompacted());
+ }
+
+ public void addTestInvocation(Integer tName, Invocation invocation) {
+ getOrAddTestsHistory(tName).innerAddInvocation(invocation);
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java
similarity index 53%
copy from tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
copy to tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java
index a7af3f8..ab5f0b9 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/SuiteHistory.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java
@@ -14,27 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.ignite.tcignited.buildref;
-package org.apache.ignite.tcignited.build;
+import com.google.common.collect.Lists;
+import java.util.Collections;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.ignite.tcservice.ITeamcity;
-import java.util.HashMap;
-import java.util.Map;
+public class BranchEquivalence {
+ /** Default synonyms. */
+ private static final List<String> DEFAULT_SYNONYMS
+ = Collections.unmodifiableList(
+ Lists.newArrayList(ITeamcity.DEFAULT, ITeamcity.REFS_HEADS_MASTER, ITeamcity.MASTER));
-import java.util.TreeMap;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
-import org.apache.ignite.internal.binary.BinaryObjectExImpl;
-import org.apache.ignite.tcignited.history.SuiteInvocation;
-/**
- * Suite run history summary.
- */
-public class SuiteHistory {
- /** Tests history: Test name ID->RunHistory */
- Map<Integer, RunHistCompacted> testsHistory = new HashMap<>();
-
- public int size(Ignite ignite) {
- BinaryObjectExImpl binary = ignite.binary().toBinary(this);
- return binary.length();
+ public List<String> branchForQuery(@Nullable String branchName) {
+ if (ITeamcity.DEFAULT.equals(branchName))
+ return DEFAULT_SYNONYMS;
+ else
+ return Collections.singletonList(branchName);
}
+
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
index f3a3e89..e8c8ae0 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
@@ -175,7 +175,7 @@ public class BuildRefDao {
*/
@AutoProfiling
@Nonnull public List<BuildRefCompacted> getAllBuildsCompacted(int srvId,
- String buildTypeId,
+ String buildTypeId,
List<String> bracnhNameQry) {
Integer buildTypeIdId = compactor.getStringIdIfPresent(buildTypeId);
if (buildTypeIdId == null)
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
new file mode 100644
index 0000000..68743b7
--- /dev/null
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.tcignited.history;
+
+import com.google.common.cache.CacheBuilder;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
+import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
+import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
+import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
+import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
+import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.build.FatBuildDao;
+import org.apache.ignite.tcignited.build.SuiteHistory;
+import org.apache.ignite.tcignited.buildref.BranchEquivalence;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
+
+/**
+ *
+ */
+public class HistoryCollector {
+ /** History DAO. */
+ @Inject private SuiteInvocationHistoryDao histDao;
+
+ /** Fat build DAO. */
+ @Inject private FatBuildDao fatBuildDao;
+
+ /** Build reference DAO. */
+ @Inject private BuildRefDao buildRefDao;
+
+ /** Compactor. */
+ @Inject private IStringCompactor compactor;
+
+ /** Ignite provider. */
+ @Inject private Provider<Ignite> igniteProvider;
+
+ /** Branch equivalence. */
+ @Inject private BranchEquivalence branchEquivalence;
+
+ /** Run history DAO. */
+ @Inject private RunHistCompactedDao runHistCompactedDao;
+
+ /**
+ * Non persistence cache for all suite RunHistory for particular branch. RunHistKey(ServerId||BranchId||suiteId)->
+ * Build reference
+ */
+ private final com.google.common.cache.Cache<RunHistKey, SuiteHistory> runHistInMemCache
+ = CacheBuilder.newBuilder()
+ .maximumSize(8000)
+ .expireAfterAccess(16, TimeUnit.MINUTES)
+ .softValues()
+ .build();
+
+ /**
+ * @param srvIdMaskHigh Server id mask to be placed at high bits in the key.
+ * @param testName Test name.
+ * @param buildTypeId Suite (Build type) id.
+ * @param normalizedBaseBranch Branch name.
+ */
+ @AutoProfiling
+ public IRunHistory getTestRunHist(int srvIdMaskHigh, int testName, int buildTypeId,
+ int normalizedBaseBranch) {
+
+ RunHistKey runHistKey = new RunHistKey(srvIdMaskHigh, buildTypeId, normalizedBaseBranch);
+
+ SuiteHistory hist;
+ try {
+ hist = runHistInMemCache.get(runHistKey,
+ () -> loadSuiteHistory(srvIdMaskHigh, buildTypeId, normalizedBaseBranch));
+ }
+ catch (ExecutionException e) {
+ throw ExceptionUtil.propagateException(e);
+ }
+
+ return hist.getTestRunHist(testName);
+ }
+
+
+ /**
+ * Latest actual Build ids supplier. This supplier should handle all equivalent branches in
+ * it.
+ * @param srvId
+ * @param buildTypeId
+ * @param normalizedBaseBranch
+ * @param knownBuilds Known builds.
+ */
+ @AutoProfiling
+ protected Set<Integer> determineLatestBuildsFunction(
+ int srvId, int buildTypeId, int normalizedBaseBranch, Set<Integer> knownBuilds) {
+ String btId = compactor.getStringFromId(buildTypeId);
+ String branchId = compactor.getStringFromId(normalizedBaseBranch);
+ List<BuildRefCompacted> compacted = buildRefDao.getAllBuildsCompacted(srvId, btId,
+ branchEquivalence.branchForQuery(branchId));
+ long curTs = System.currentTimeMillis();
+ Set<Integer> buildIds = compacted.stream()
+ .filter(b -> !knownBuilds.contains(b.id()))
+ //todo filter queued, cancelled and so on
+ .filter(
+ bRef -> {
+ //todo getAll
+ //todo getStartTime From FatBuild
+ Long startTime = runHistCompactedDao.getBuildStartTime(srvId, bRef.id());
+ if (startTime == null)
+ return false;
+
+ return Duration.ofMillis(curTs - startTime).toDays() < InvocationData.MAX_DAYS;
+ }
+ ).map(BuildRefCompacted::id).collect(Collectors.toSet());
+
+ System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " + buildIds.size());
+
+ return buildIds;
+ }
+
+ @AutoProfiling
+ public SuiteHistory loadSuiteHistory(int srvId,
+ int buildTypeId,
+ int normalizedBaseBranch) {
+ Map<Integer, SuiteInvocation> suiteRunHist = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
+
+ Set<Integer> buildIds = determineLatestBuildsFunction(srvId, buildTypeId, normalizedBaseBranch, suiteRunHist.keySet());
+
+ HashSet<Integer> missedBuildsIds = new HashSet<>(buildIds);
+
+ missedBuildsIds.removeAll(suiteRunHist.keySet());
+
+ if (!missedBuildsIds.isEmpty()) {
+ Map<Integer, SuiteInvocation> addl = addSuiteInvocationsToHistory(srvId, missedBuildsIds, normalizedBaseBranch);
+
+ System.err.println("***** + Adding to persisted history for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch " + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
+ addl.size() + " invocations");
+
+ histDao.putAll(srvId, addl);
+ suiteRunHist.putAll(addl);
+ }
+
+ SuiteHistory sumary = new SuiteHistory();
+
+ suiteRunHist.forEach((buildId, suiteInv) -> suiteInv.tests().forEach(sumary::addTestInvocation));
+
+ System.err.println("***** History for suite "
+ + compactor.getStringFromId(buildTypeId)
+ + " branch" + compactor.getStringFromId(normalizedBaseBranch) + " requires " +
+ sumary.size(igniteProvider.get()) + " bytes");
+
+ return sumary;
+ }
+
+ public void invalidateHistoryInMem(int srvId, Stream<BuildRefCompacted> stream) {
+ Iterable<RunHistKey> objects =
+ stream
+ .map(b -> new RunHistKey(srvId, b.buildTypeId(), b.branchName()))
+ .collect(Collectors.toSet());
+
+ runHistInMemCache.invalidateAll(objects);
+ }
+
+
+
+
+ @AutoProfiling
+ public Map<Integer, SuiteInvocation> addSuiteInvocationsToHistory(int srvId,
+ HashSet<Integer> missedBuildsIds, int normalizedBaseBranch) {
+ Map<Integer, SuiteInvocation> suiteRunHist = new HashMap<>();
+ int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
+
+ fatBuildDao.getAllFatBuilds(srvId, missedBuildsIds).forEach((buildCacheKey, fatBuildCompacted) -> {
+ SuiteInvocation sinv = new SuiteInvocation(srvId, normalizedBaseBranch, fatBuildCompacted, compactor, (k, v) -> false);
+
+ Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
+ tests.forEach(
+ testCompacted -> {
+ Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
+
+ sinv.addTest(testCompacted.testName(), invocation);
+ }
+ );
+
+ suiteRunHist.put(fatBuildCompacted.id(), sinv);
+ });
+
+ return suiteRunHist;
+ }
+
+ /*
+ @AutoProfiling
+ protected SuiteHistory calcSuiteHistory(int srvIdMaskHigh, Set<Integer> buildIds) {
+ Set<Long> cacheKeys = buildsIdsToCacheKeys(srvIdMaskHigh, buildIds);
+
+ int successStatusStrId = compactor.getStringId(TestOccurrence.STATUS_SUCCESS);
+
+ CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> processor = new FatBuildDao.HistoryCollectProcessor(successStatusStrId);
+
+ Map<Long, EntryProcessorResult<Map<Integer, Invocation>>> map = buildsCache.invokeAll(cacheKeys, processor);
+
+ SuiteHistory hist = new SuiteHistory();
+
+ map.values().forEach(
+ res -> {
+ if (res == null)
+ return;
+
+ Map<Integer, Invocation> invocationMap = res.get();
+
+ if (invocationMap == null)
+ return;
+
+ invocationMap.forEach((k, v) -> {
+ RunHistCompacted compacted = hist.testsHistory.computeIfAbsent(k,
+ k_ -> new RunHistCompacted());
+
+ compacted.innerAddInvocation(v);
+ });
+
+ }
+ );
+
+ System.err.println("Suite history: tests in scope "
+ + hist.testsHistory.size()
+ + " for " +buildIds.size() + " builds checked"
+ + " size " + hist.size(igniteProvider.get()));
+
+ return hist;
+ }
+ */
+
+
+ private static class HistoryCollectProcessor implements CacheEntryProcessor<Long, FatBuildCompacted, Map<Integer, Invocation>> {
+ private final int successStatusStrId;
+
+ public HistoryCollectProcessor(int successStatusStrId) {
+ this.successStatusStrId = successStatusStrId;
+ }
+
+ @Override public Map<Integer, Invocation> process(MutableEntry<Long, FatBuildCompacted> entry,
+ Object... arguments) throws EntryProcessorException {
+ if (entry.getValue() == null)
+ return null;
+
+ Map<Integer, Invocation> hist = new HashMap<>();
+ FatBuildCompacted fatBuildCompacted = entry.getValue();
+ Stream<TestCompacted> tests = fatBuildCompacted.getAllTests();
+ tests.forEach(
+ testCompacted -> {
+ Invocation invocation = testCompacted.toInvocation(fatBuildCompacted, (k, v) -> false, successStatusStrId);
+
+ hist.put(testCompacted.testName(), invocation);
+ }
+ );
+
+ return hist;
+ }
+ }
+
+}