You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/11/27 17:26:29 UTC
[ignite-teamcity-bot] branch ignite-9542-new-run-stripe updated:
IGNITE-9542 Build start time cache is now used for duplicates protection
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9542-new-run-stripe
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9542-new-run-stripe by this push:
new f940ba9 IGNITE-9542 Build start time cache is now used for duplicates protection
f940ba9 is described below
commit f940ba99a24972c55b7a289377e753f2e666ad51
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Tue Nov 27 20:26:27 2018 +0300
IGNITE-9542 Build start time cache is now used for duplicates protection
---
.../ignite/ci/di/scheduler/TcBotScheduler.java | 17 +++--
.../ci/teamcity/ignited/ITeamcityIgnited.java | 2 -
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 2 +-
.../ignited/runhist/RunHistCompactedDao.java | 26 ++++++-
.../ci/teamcity/ignited/runhist/RunHistSync.java | 80 +++++++++++++---------
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 34 ++++-----
6 files changed, 96 insertions(+), 65 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
index c931b08..e83e523 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ci.di.scheduler;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import org.apache.ignite.ci.di.MonitoredTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,7 +29,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
class TcBotScheduler implements IScheduler {
- public static final int POOL_SIZE = 10;
+ public static final int POOL_SIZE = 16;
/** Logger. */
private static final Logger logger = LoggerFactory.getLogger(TcBotScheduler.class);
@@ -50,22 +51,20 @@ class TcBotScheduler implements IScheduler {
task.sheduleWithQuitePeriod(cmd, queitPeriod, unit);
if (tickGuard.compareAndSet(false, true)) {
- for (int i = 0; i < POOL_SIZE; i++) {
- int threadNo = i;
-
+ for (int threadId = 0; threadId < POOL_SIZE; threadId++) {
+ String threadNme = ", runner" + Strings.padStart(Integer.toString(threadId), 2, '0');
int period = 15000 + ThreadLocalRandom.current().nextInt(10000);
- service().scheduleAtFixedRate(() -> checkNamedTasks(threadNo), 0, period, TimeUnit.MILLISECONDS);
+ service().scheduleAtFixedRate(() -> checkNamedTasks(threadNme), 0, period, TimeUnit.MILLISECONDS);
}
}
}
/**
- *
- * @param threadNo
+ * @param threadNme
*/
@SuppressWarnings({"UnusedReturnValue", "WeakerAccess"})
- @MonitoredTask(name = "Scheduled, runner", nameExtArgIndex = 0)
- protected String checkNamedTasks(int threadNo) {
+ @MonitoredTask(name = "Scheduled", nameExtArgIndex = 0)
+ protected String checkNamedTasks(String threadNme) {
AtomicInteger run = new AtomicInteger();
List<Throwable> problems = new ArrayList<>();
namedTasks.forEach((s, task) -> {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
index d2280a1..f1f6bb2 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
@@ -107,8 +107,6 @@ public interface ITeamcityIgnited {
return getFatBuild(id, SyncMode.RELOAD_QUEUED);
}
-
-
/**
* @param id Id.
* @param mode Refresh mode.
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index e1022cd..4962f4c 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -407,7 +407,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return existingBuild;
buildRefDao.save(srvIdMaskHigh, new BuildRefCompacted(savedVer));
- runHistSync.saveToHistoryLater(srvIdMaskHigh, buildId, savedVer);
+ runHistSync.saveToHistoryLater(srvNme, buildId, savedVer);
return savedVer;
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
index 46768db..922742a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
@@ -31,9 +31,12 @@ import javax.inject.Provider;
import java.util.Collections;
public class RunHistCompactedDao {
- /** Cache name*/
+ /** Cache name.*/
public static final String TEST_HIST_CACHE_NAME = "testRunHistV0";
+ /** Build Start time Cache name. */
+ public static final String BUILD_START_TIME_CACHE_NAME = "buildStartTimeV0";
+
/** Ignite provider. */
@Inject
private Provider<Ignite> igniteProvider;
@@ -42,7 +45,7 @@ public class RunHistCompactedDao {
private IgniteCache<RunHistKey, RunHistCompacted> testHistCache;
/** Build start time. */
- private IgniteCache<Integer, Long> buildStartTime;
+ private IgniteCache<Long, Long> buildStartTime;
/** Compactor. */
@Inject private IStringCompactor compactor;
@@ -58,6 +61,8 @@ public class RunHistCompactedDao {
cfg.setQueryEntities(Collections.singletonList(new QueryEntity(RunHistKey.class, RunHistCompacted.class)));
testHistCache = ignite.getOrCreateCache(cfg);
+
+ buildStartTime = ignite.getOrCreateCache(TcHelperDb.getCacheV2Config(BUILD_START_TIME_CACHE_NAME));
}
public IRunHistory getTestRunHist(int srvIdMaskHigh, String name, String branch) {
@@ -98,4 +103,21 @@ public class RunHistCompactedDao {
);
}
+
+ /**
+ * @param srvId Server id mask high.
+ * @param buildId Build id.
+ */
+ public static long buildIdToCacheKey(long srvId, int buildId) {
+ return (long)buildId | srvId << 32;
+ }
+
+
+ public boolean buildWasProcessed(int srvId, int buildId) {
+ return buildStartTime.containsKey(buildIdToCacheKey(srvId, buildId));
+ }
+
+ public boolean setBuildProcessed(int srvId, int buildId, long ts) {
+ return buildStartTime.putIfAbsent(buildIdToCacheKey(srvId, buildId), ts);
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
index 1408f80..a14246f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
@@ -18,10 +18,13 @@
package org.apache.ignite.ci.teamcity.ignited.runhist;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-
+import javax.annotation.concurrent.GuardedBy;
+import javax.inject.Inject;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.di.MonitoredTask;
import org.apache.ignite.ci.di.scheduler.IScheduler;
@@ -31,22 +34,24 @@ import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
-
-import javax.annotation.concurrent.GuardedBy;
-import javax.inject.Inject;
-import java.util.HashMap;
-import java.util.Map;
import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
*/
public class RunHistSync {
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(RunHistSync.class);
+
/** Compactor. */
@Inject private IStringCompactor compactor;
+ /** Scheduler. */
@Inject private IScheduler scheduler;
+ /** Run History DAO. */
@Inject private RunHistCompactedDao histDao;
/** Build reference DAO. */
@@ -55,38 +60,42 @@ public class RunHistSync {
/** Build DAO. */
@Inject private FatBuildDao fatBuildDao;
-
/** Build to save to history. */
@GuardedBy("this")
- private final Map<Integer, SyncTask> buildToSave = new HashMap<>();
+ private final Map<String, SyncTask> buildToSave = new HashMap<>();
/**
- * @param srvId Server id.
+ * @param srvVame Server id.
* @param buildId Build id.
* @param build Build.
*/
- public void saveToHistoryLater(int srvId, int buildId, FatBuildCompacted build) {
+ public void saveToHistoryLater(String srvVame, int buildId, FatBuildCompacted build) {
if (!validForStatistics(build))
return;
+ int srvId = ITeamcityIgnited.serverIdToInt(srvVame);
+ if (histDao.buildWasProcessed(srvId, buildId))
+ return;
+
synchronized (this) {
- final SyncTask syncTask = buildToSave.computeIfAbsent(srvId, s -> new SyncTask());
+ final SyncTask syncTask = buildToSave.computeIfAbsent(srvVame, s -> new SyncTask());
syncTask.buildsToSave.putIfAbsent(build.id(), build);
}
- scheduler.sheduleNamed(RunHistSync.class.getSimpleName() + ".saveBuildToHistory",
- () -> saveBuildToHistory(srvId), 2, TimeUnit.MINUTES);
+ scheduler.sheduleNamed(taskName("saveBuildToHistory", srvVame),
+ () -> saveBuildToHistory(srvVame), 2, TimeUnit.MINUTES);
}
@AutoProfiling
- @MonitoredTask(name="Save Builds To History")
+ @MonitoredTask(name="Save Builds To History", nameExtArgIndex = 0)
@SuppressWarnings("WeakerAccess")
- protected String saveBuildToHistory(int srvId) {
+ protected String saveBuildToHistory(String srvName) {
+ int srvMask = ITeamcityIgnited.serverIdToInt(srvName);
Map<Integer, FatBuildCompacted> saveThisRun;
synchronized (this) {
- final SyncTask syncTask = buildToSave.get(srvId);
+ final SyncTask syncTask = buildToSave.get(srvName);
if (syncTask == null)
return "Nothing to sync";
@@ -98,16 +107,23 @@ public class RunHistSync {
AtomicInteger builds = new AtomicInteger();
AtomicInteger invocations = new AtomicInteger();
AtomicInteger duplicates = new AtomicInteger();
+
saveThisRun.values().forEach(
build -> {
builds.incrementAndGet();
+ if (!histDao.setBuildProcessed(srvMask, build.id(), build.getStartDateTs())) {
+ duplicates.incrementAndGet();
+
+ return;
+ }
+
build.getAllTests().forEach(t -> {
Invocation inv = t.toInvocation(compactor, build);
- final Boolean res = histDao.addInvocation(srvId, t, build.id(), build.branchName(), inv);
+ final Boolean res = histDao.addInvocation(srvMask, t, build.id(), build.branchName(), inv);
- if(Boolean.FALSE.equals(res))
+ if (Boolean.FALSE.equals(res))
duplicates.incrementAndGet();
else
invocations.incrementAndGet();
@@ -120,8 +136,8 @@ public class RunHistSync {
}
public void invokeLaterFindMissingHistory(String srvName) {
- scheduler.sheduleNamed(taskName("findMissingHistFromBuildRef", srvName),
- () -> findMissingHistFromBuildRef(srvName), 360, TimeUnit.MINUTES);
+ scheduler.sheduleNamed(taskName("findMissingHistFromBuildRef", srvName),
+ () -> findMissingHistFromBuildRef(srvName), 360, TimeUnit.MINUTES);
}
@NotNull
@@ -130,9 +146,9 @@ public class RunHistSync {
}
@SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
- @MonitoredTask(name = "Find missing Build History", nameExtArgsIndexes = {0})
+ @MonitoredTask(name = "Find Missing Build History", nameExtArgsIndexes = {0})
@AutoProfiling
- protected String findMissingHistFromBuildRef(String srvId ) {
+ protected String findMissingHistFromBuildRef(String srvId) {
int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
final int[] buildRefKeys = buildRefDao.getAllIds(srvIdMaskHigh);
@@ -146,7 +162,7 @@ public class RunHistSync {
if (buildsIdsToLoad.size() >= 100) {
totalAskedToLoad += buildsIdsToLoad.size();
- scheduleHistLoad(srvIdMaskHigh, buildsIdsToLoad);
+ scheduleHistLoad(srvId, buildsIdsToLoad);
buildsIdsToLoad.clear();
}
buildsIdsToLoad.add(buildRefKey);
@@ -154,29 +170,31 @@ public class RunHistSync {
if (!buildsIdsToLoad.isEmpty()) {
totalAskedToLoad += buildsIdsToLoad.size();
- scheduleHistLoad(srvIdMaskHigh, buildsIdsToLoad);
+ scheduleHistLoad(srvId, buildsIdsToLoad);
}
return "Invoked later load for " + totalAskedToLoad + " builds from " + srvId;
}
- private void scheduleHistLoad(int srvIdMaskHigh, List<Integer> load) {
- //todo implement
- System.err.println("scheduleHistLoad: " + load.toString());
-
+ /**
+ * @param srvNme Server name;
+ * @param load Build IDs to be loaded into history cache later.
+ */
+ private void scheduleHistLoad(String srvNme, List<Integer> load) {
load.forEach(id -> {
- FatBuildCompacted fatBuild = fatBuildDao.getFatBuild(srvIdMaskHigh, id);
+ FatBuildCompacted fatBuild = fatBuildDao.getFatBuild(ITeamcityIgnited.serverIdToInt(srvNme), id);
if (validForStatistics(fatBuild))
- saveToHistoryLater(srvIdMaskHigh, fatBuild.id(), fatBuild);
+ saveToHistoryLater(srvNme, fatBuild.id(), fatBuild);
else
- System.err.println("Build is not valid for stat: " + fatBuild.toString());
+ logger.info("Build is not valid for stat: " + fatBuild.toString());
});
}
private boolean validForStatistics(FatBuildCompacted fatBuild) {
return fatBuild != null
&& !fatBuild.isFakeStub()
+ && !fatBuild.isOutdatedEntityVersion()
&& !fatBuild.isCancelled(compactor)
&& fatBuild.isFinished(compactor);
}
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index 0a5e013..24ee9a7 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -49,16 +49,13 @@ import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
-import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompactedDao;
-import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistSync;
import org.apache.ignite.ci.teamcity.pure.BuildHistoryEmulator;
import org.apache.ignite.ci.teamcity.pure.ITeamcityHttpConnection;
import org.apache.ignite.ci.user.ICredentialsProv;
import org.apache.ignite.ci.util.XmlUtil;
import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.jetbrains.annotations.NotNull;
import org.junit.AfterClass;
@@ -348,14 +345,7 @@ public class IgnitedTcInMemoryIntegrationTest {
@Test
public void testRunHistSaveLoad() {
-
- Injector injector = Guice.createInjector(new TeamcityIgnitedModule(), new AbstractModule() {
- @Override
- protected void configure() {
- bind(Ignite.class).toInstance(ignite);
- bind(IScheduler.class).to(DirectExecNoWaitSheduler.class).in(new SingletonScope());
- }
- });
+ Injector injector = Guice.createInjector(new TeamcityIgnitedModule(), new IgniteAndShedulerTestModule());
injector.getInstance(RunHistCompactedDao.class).init();
final IStringCompactor c = injector.getInstance(IStringCompactor.class);
@@ -370,7 +360,7 @@ public class IgnitedTcInMemoryIntegrationTest {
final Map<Integer, FatBuildCompacted> buildsMap = tst.apacheBuilds();
final RunHistSync histSync = injector.getInstance(RunHistSync.class);
- buildsMap.forEach((id, build) -> histSync.saveToHistoryLater(ITeamcityIgnited.serverIdToInt(srvId), id, build));
+ buildsMap.forEach((id, build) -> histSync.saveToHistoryLater(srvId, id, build));
final ITeamcityIgnitedProvider inst = injector.getInstance(ITeamcityIgnitedProvider.class);
final ITeamcityIgnited srv = inst.server(srvId, Mockito.mock(ICredentialsProv.class));
@@ -381,16 +371,9 @@ public class IgnitedTcInMemoryIntegrationTest {
}
-
@Test
public void testHistoryBackgroundUpdateWorks() {
- Injector injector = Guice.createInjector(new TeamcityIgnitedModule(), new AbstractModule() {
- @Override
- protected void configure() {
- bind(Ignite.class).toInstance(ignite);
- bind(IScheduler.class).to(DirectExecNoWaitSheduler.class).in(new SingletonScope());
- }
- });
+ Injector injector = Guice.createInjector(new TeamcityIgnitedModule(), new IgniteAndShedulerTestModule());
injector.getInstance(RunHistCompactedDao.class).init();
@@ -429,4 +412,15 @@ public class IgnitedTcInMemoryIntegrationTest {
assertNotNull(testRunHist);
assertEquals(0.5, testRunHist.getFailRate(), 0.1);
}
+
+ /**
+ *
+ */
+ private static class IgniteAndShedulerTestModule extends AbstractModule {
+ /** {@inheritDoc} */
+ @Override protected void configure() {
+ bind(Ignite.class).toInstance(ignite);
+ bind(IScheduler.class).to(DirectExecNoWaitSheduler.class).in(new SingletonScope());
+ }
+ }
}