You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/25 17:48:10 UTC
[ignite-teamcity-bot] branch ignite-9848-load-all-builds updated:
First proactive loading implementation for all builds
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-load-all-builds
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9848-load-all-builds by this push:
new d2886db First proactive loading implementation for all builds
d2886db is described below
commit d2886dbc0a0b013d7fe902ce74bb0044806a0509
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Oct 25 20:47:56 2018 +0300
First proactive loading implementation for all builds
---
.../ignite/ci/di/scheduler/TcBotScheduler.java | 9 +-
.../ignite/ci/teamcity/ignited/BuildRefDao.java | 12 ++-
.../ignite/ci/teamcity/ignited/FatBuildDao.java | 64 ++++---------
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 106 ++++++++++++++++++---
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 5 +-
5 files changed, 130 insertions(+), 66 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
index 40508e8..6554358 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
@@ -27,8 +27,9 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ignite.ci.di.MonitoredTask;
class TcBotScheduler implements IScheduler {
+ public static final int POOL_SIZE = 3;
/** Executor service. */
- private volatile ScheduledExecutorService executorSvc = Executors.newScheduledThreadPool(3);
+ private volatile ScheduledExecutorService executorSvc = Executors.newScheduledThreadPool(POOL_SIZE);
/** Submit named task checker guard. */
private AtomicBoolean tickGuard = new AtomicBoolean();
@@ -44,8 +45,10 @@ class TcBotScheduler implements IScheduler {
task.sheduleWithQuitePeriod(cmd, queitPeriod, unit);
- if (tickGuard.compareAndSet(false, true))
- service().scheduleAtFixedRate(this::checkNamedTasks, 0, 1, TimeUnit.SECONDS);
+ if (tickGuard.compareAndSet(false, true)) {
+ for (int i = 0; i < POOL_SIZE; i++)
+ service().scheduleAtFixedRate(this::checkNamedTasks, 0, 5, TimeUnit.SECONDS);
+ }
}
/**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
index a018654..1e38c73 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
@@ -65,7 +65,7 @@ public class BuildRefDao {
* @param srvId Server id mask high.
* @param ghData Gh data.
*/
- public int saveChunk(long srvId, List<BuildRef> ghData) {
+ public Set<Long> saveChunk(long srvId, List<BuildRef> ghData) {
Set<Long> ids = ghData.stream().map(BuildRef::getId)
.filter(Objects::nonNull)
.map(buildId -> buildIdToCacheKey(srvId, buildId))
@@ -89,17 +89,23 @@ public class BuildRefDao {
int size = entriesToPut.size();
if (size != 0)
buildsCache.putAll(entriesToPut);
- return size;
+
+ return entriesToPut.keySet();
}
/**
* @param srvId Server id mask high.
* @param buildId Build id.
*/
- private long buildIdToCacheKey(long srvId, int buildId) {
+ public long buildIdToCacheKey(long srvId, int buildId) {
return (long)buildId | srvId << 32;
}
+ public static int cacheKeyToBuildId(Long cacheKey) {
+ long l = cacheKey << 32;
+ return (int) (l>>32);
+ }
+
/**
* @param srvId Server id mask high.
* @param buildTypeId Build type id.
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/FatBuildDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/FatBuildDao.java
index e3d8a33..01ad032 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/FatBuildDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/FatBuildDao.java
@@ -17,23 +17,21 @@
package org.apache.ignite.ci.teamcity.ignited;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import java.util.TreeMap;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.ci.db.TcHelperDb;
-import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrences;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.jetbrains.annotations.Nullable;
/**
*
@@ -61,23 +59,22 @@ public class FatBuildDao {
/**
* @param srvIdMaskHigh Server id mask high.
* @param build Build data.
- * @param tests TestOccurrences.
+ * @param tests TestOccurrences one or several pages.
+ * @param existingBuild existing version of build in the DB.
* @return Fat Build saved (if modifications detected)
*/
public FatBuildCompacted saveBuild(long srvIdMaskHigh,
Build build,
- List<TestOccurrencesFull> tests) {
-
- long cacheKey = buildIdToCacheKey(srvIdMaskHigh, build.getId());
- FatBuildCompacted existingEntry = buildsCache.get(cacheKey);
+ List<TestOccurrencesFull> tests,
+ @Nullable FatBuildCompacted existingBuild) {
FatBuildCompacted newBuild = new FatBuildCompacted(compactor, build);
for (TestOccurrencesFull next : tests)
newBuild.addTests(compactor, next.getTests());
- if (existingEntry == null || !existingEntry.equals(newBuild)) {
- buildsCache.put(cacheKey, newBuild);
+ if (existingBuild == null || !existingBuild.equals(newBuild)) {
+ buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, build.getId()), newBuild);
return newBuild;
}
@@ -86,49 +83,30 @@ public class FatBuildDao {
/**
* @param srvIdMaskHigh Server id mask high.
- * @param buildData Gh data.
+ * @param buildId Build id.
*/
- public int saveChunk(long srvIdMaskHigh, List<Build> buildData) {
- Set<Long> ids = buildData.stream().map(BuildRef::getId)
- .filter(Objects::nonNull)
- .map(buildId -> buildIdToCacheKey(srvIdMaskHigh, buildId))
- .collect(Collectors.toSet());
-
- Map<Long, FatBuildCompacted> existingEntries = buildsCache.getAll(ids);
- Map<Long, FatBuildCompacted> entriesToPut = new TreeMap<>();
-
- List<FatBuildCompacted> collect = buildData.stream()
- .map(ref -> new FatBuildCompacted(compactor, ref))
- .collect(Collectors.toList());
-
- for (FatBuildCompacted next : collect) {
- long cacheKey = buildIdToCacheKey(srvIdMaskHigh, next.id());
- FatBuildCompacted buildPersisted = existingEntries.get(cacheKey);
-
- if (buildPersisted == null || !buildPersisted.equals(next))
- entriesToPut.put(cacheKey, next);
- }
-
- int size = entriesToPut.size();
- if (size != 0)
- buildsCache.putAll(entriesToPut);
-
- return size;
+ public static long buildIdToCacheKey(long srvIdMaskHigh, int buildId) {
+ return (long)buildId | srvIdMaskHigh << 32;
}
/**
* @param srvIdMaskHigh Server id mask high.
* @param buildId Build id.
*/
- private long buildIdToCacheKey(long srvIdMaskHigh, int buildId) {
- return (long)buildId | srvIdMaskHigh << 32;
+ public FatBuildCompacted getFatBuild(int srvIdMaskHigh, int buildId) {
+ return buildsCache.get(buildIdToCacheKey(srvIdMaskHigh, buildId));
}
/**
* @param srvIdMaskHigh Server id mask high.
- * @param buildId Build id.
+ * @param buildsIds Builds ids.
*/
- public FatBuildCompacted getFatBuild(int srvIdMaskHigh, int buildId) {
- return buildsCache.get(buildIdToCacheKey(srvIdMaskHigh, buildId));
+ public Map<Long, FatBuildCompacted> getAllFatBuilds(int srvIdMaskHigh, Collection<Integer> buildsIds ) {
+ Set<Long> ids = buildsIds.stream()
+ .filter(Objects::nonNull)
+ .map(buildId -> buildIdToCacheKey(srvIdMaskHigh, buildId))
+ .collect(Collectors.toSet());
+
+ return buildsCache.getAll(ids);
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 6b728c7..14a0e4d 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -19,13 +19,18 @@ package org.apache.ignite.ci.teamcity.ignited;
import com.google.common.base.Strings;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
+import javax.annotation.concurrent.GuardedBy;
import javax.inject.Inject;
import com.google.common.collect.Sets;
import org.apache.ignite.ci.ITeamcity;
@@ -34,12 +39,18 @@ import org.apache.ignite.ci.di.MonitoredTask;
import org.apache.ignite.ci.di.scheduler.IScheduler;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrences;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TeamcityIgnitedImpl implements ITeamcityIgnited {
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(TestCompacted.class);
+
/** Server id. */
private String srvId;
@@ -58,6 +69,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
/** Server ID mask for cache Entries. */
private int srvIdMaskHigh;
+ @GuardedBy("this")
+ private Set<Integer> buildToLoad = new HashSet<>();
public void init(String srvId, ITeamcityConn conn) {
this.srvId = srvId;
@@ -68,6 +81,18 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
fatBuildDao.init();
}
+ public void scheduleBuildsLoad(List<Integer> buildsToAskFromTc) {
+ synchronized (this) {
+ buildToLoad.addAll(buildsToAskFromTc);
+ }
+
+ scheduler.sheduleNamed(taskName("loadFatBuilds"), this::loadFatBuilds, 2, TimeUnit.MINUTES);
+ }
+
+ @NotNull public String taskName(String taskName) {
+ return ITeamcityIgnited.class.getSimpleName() +"." + taskName + "." + srvId;
+ }
+
/** {@inheritDoc} */
@Override public String host() {
return conn.host();
@@ -78,8 +103,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
@Override public List<BuildRef> getBuildHistory(
@Nullable String buildTypeId,
@Nullable String branchName) {
- scheduler.sheduleNamed(ITeamcityIgnited.class.getSimpleName() + ".actualizeRecentBuilds",
- this::actualizeRecentBuilds, 2, TimeUnit.MINUTES);
+ scheduler.sheduleNamed(taskName("actualizeRecentBuilds"), this::actualizeRecentBuilds, 2, TimeUnit.MINUTES);
String bracnhNameQry ;
if (ITeamcity.DEFAULT.equals(branchName))
@@ -101,10 +125,22 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
@Override public FatBuildCompacted getFatBuild(int buildId) {
- FatBuildCompacted buildPersisted = fatBuildDao.getFatBuild(srvIdMaskHigh, buildId);
- if (buildPersisted != null && !buildPersisted.isOutdatedEntityVersion())
- return buildPersisted;
+ FatBuildCompacted existingBuild = fatBuildDao.getFatBuild(srvIdMaskHigh, buildId);
+ if (existingBuild != null && !existingBuild.isOutdatedEntityVersion())
+ return existingBuild;
+
+ FatBuildCompacted savedVer = reloadBuild(buildId, existingBuild);
+ return savedVer == null ? existingBuild : savedVer;
+ }
+
+ /**
+ * @param buildId
+ * @param existingBuild
+ * @return new build if it was updated or null if no updates detected
+ */
+ public FatBuildCompacted reloadBuild(int buildId, FatBuildCompacted existingBuild) {
+ // System.err.println(Thread.currentThread().getName()+ ": Build " + buildId);
//todo some sort of locking to avoid double requests
Build build = conn.getBuild(buildId);
@@ -119,7 +155,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
while (!Strings.isNullOrEmpty(nextHref));
- return fatBuildDao.saveBuild(srvIdMaskHigh, build, tests);
+ return fatBuildDao.saveBuild(srvIdMaskHigh, build, tests, existingBuild);
}
/**
@@ -140,8 +176,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
*
*/
private void sheduleResync() {
- scheduler.sheduleNamed(ITeamcityIgnited.class.getSimpleName() + ".fullReindex",
- this::fullReindex, 120, TimeUnit.MINUTES);
+ scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 120, TimeUnit.MINUTES);
}
/**
@@ -163,7 +198,10 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
AtomicReference<String> outLinkNext = new AtomicReference<>();
List<BuildRef> tcDataFirstPage = conn.getBuildRefs(null, outLinkNext);
- int cntSaved = buildRefDao.saveChunk(srvIdMaskHigh, tcDataFirstPage);
+ Set<Long> buildsUpdated = buildRefDao.saveChunk(srvIdMaskHigh, tcDataFirstPage);
+ int totalUpdated = buildsUpdated.size();
+ scheduleBuildsLoad(cacheKeysToBuildIds(buildsUpdated));
+
int totalChecked = tcDataFirstPage.size();
final Set<Integer> stillNeedToFind =
@@ -173,9 +211,12 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
String nextPageUrl = outLinkNext.get();
outLinkNext.set(null);
List<BuildRef> tcDataNextPage = conn.getBuildRefs(nextPageUrl, outLinkNext);
- int savedCurChunk = buildRefDao.saveChunk(srvIdMaskHigh, tcDataNextPage);
+ Set<Long> curChunkBuildsSaved = buildRefDao.saveChunk(srvIdMaskHigh, tcDataNextPage);
+ totalUpdated += curChunkBuildsSaved.size();
+ scheduleBuildsLoad(cacheKeysToBuildIds(curChunkBuildsSaved));
+
+ int savedCurChunk = curChunkBuildsSaved.size();
- cntSaved += savedCurChunk;
totalChecked += tcDataNextPage.size();
if (!fullReindex) {
@@ -187,7 +228,46 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
}
- return "Entries saved " + cntSaved + " Builds checked " + totalChecked;
+ return "Entries saved " + totalUpdated + " Builds checked " + totalChecked;
}
+ @NotNull private List<Integer> cacheKeysToBuildIds(Collection<Long> cacheKeysUpdated) {
+ return cacheKeysUpdated.stream().map(BuildRefDao::cacheKeyToBuildId).collect(Collectors.toList());
+ }
+
+ private void loadFatBuilds() {
+ Set<Integer> load;
+ synchronized (this) {
+ load = buildToLoad;
+ buildToLoad = new HashSet<>();
+ }
+ doLoadBuilds(srvId, load);
+ }
+
+ @MonitoredTask(name = "Proactive Builds Loading", nameExtArgIndex = 0)
+ @AutoProfiling
+ protected String doLoadBuilds(String srvId, Set<Integer> load) {
+ AtomicInteger err = new AtomicInteger();
+ AtomicInteger ld = new AtomicInteger();
+
+ Map<Long, FatBuildCompacted> builds = fatBuildDao.getAllFatBuilds(srvIdMaskHigh, load);
+
+ load.forEach(
+ buildId -> {
+ try {
+ FatBuildCompacted existingBuild = builds.get(FatBuildDao.buildIdToCacheKey(srvIdMaskHigh, buildId));
+
+ FatBuildCompacted savedVer = reloadBuild(buildId, existingBuild);
+
+ if (savedVer != null)
+ ld.incrementAndGet();
+ }
+ catch (Exception e) {
+ logger.error("", e);
+ err.incrementAndGet();
+ }
+ }
+ );
+ return "Builds Loaded " + ld.get() + " from " + load.size() + " requested, errors: " + err;
+ }
}
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index 1543f76..aa2e7eb 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -26,7 +26,6 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
@@ -41,8 +40,6 @@ import org.apache.ignite.ci.di.scheduler.NoOpSheduler;
import org.apache.ignite.ci.tcmodel.conf.BuildType;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrenceFull;
-import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrences;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.pure.BuildHistoryEmulator;
@@ -267,7 +264,7 @@ public class IgnitedTcInMemoryIntegrationTest {
int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(APACHE);
List<TestOccurrencesFull> occurrences = Collections.singletonList(testsRef);
- FatBuildCompacted buildCompacted = stor.saveBuild(srvIdMaskHigh, refBuild, occurrences);
+ FatBuildCompacted buildCompacted = stor.saveBuild(srvIdMaskHigh, refBuild, occurrences, null);
assertNotNull(buildCompacted);
FatBuildCompacted fatBuild = stor.getFatBuild(srvIdMaskHigh, 2153237);