You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/26 17:38:06 UTC
[ignite-teamcity-bot] 02/02: Several updator's threads implemented
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-load-all-builds
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 2717617cafa3de0e9cb406474487ddd60d8f93d8
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Oct 26 20:37:48 2018 +0300
Several updator's threads implemented
---
.../ignite/ci/di/MonitoredTaskInterceptor.java | 13 +++++-----
.../ignite/ci/di/scheduler/TcBotScheduler.java | 17 ++++++++-----
.../ignite/ci/teamcity/ignited/BuildRefDao.java | 11 +++++---
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 29 ++++++++++++++++------
.../ci/teamcity/ignited/TeamcityIgnitedModule.java | 1 +
.../ignited/{ => fatbuild}/FatBuildDao.java | 3 ++-
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 1 +
7 files changed, 51 insertions(+), 24 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTaskInterceptor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTaskInterceptor.java
index a0840da..c77816a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTaskInterceptor.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/MonitoredTaskInterceptor.java
@@ -17,23 +17,22 @@
package org.apache.ignite.ci.di;
import com.google.common.base.Strings;
-import org.aopalliance.intercept.MethodInterceptor;
-import org.aopalliance.intercept.MethodInvocation;
-import org.apache.ignite.ci.util.TimeUtil;
-import org.jetbrains.annotations.NotNull;
-
import java.lang.reflect.Method;
import java.util.Collection;
import java.util.Collections;
import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+import org.apache.ignite.ci.util.TimeUtil;
+import org.jetbrains.annotations.NotNull;
public class MonitoredTaskInterceptor implements MethodInterceptor {
- private final ConcurrentMap<String, Invocation> totalTime = new ConcurrentHashMap<>();
+ private final ConcurrentMap<String, Invocation> totalTime = new ConcurrentSkipListMap<>();
public static class Invocation {
private final AtomicLong lastStartTs = new AtomicLong();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
index cedb00b..2193666 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/di/scheduler/TcBotScheduler.java
@@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
class TcBotScheduler implements IScheduler {
- public static final int POOL_SIZE = 3;
+ public static final int POOL_SIZE = 8;
/** Logger. */
private static final Logger logger = LoggerFactory.getLogger(TcBotScheduler.class);
@@ -50,17 +50,21 @@ class TcBotScheduler implements IScheduler {
task.sheduleWithQuitePeriod(cmd, queitPeriod, unit);
if (tickGuard.compareAndSet(false, true)) {
- for (int i = 0; i < POOL_SIZE; i++)
- service().scheduleAtFixedRate(this::checkNamedTasks, 0, 5, TimeUnit.SECONDS);
+ for (int i = 0; i < POOL_SIZE; i++) {
+ int threadNo = i;
+
+ service().scheduleAtFixedRate(() -> checkNamedTasks(threadNo), 0, 20, TimeUnit.SECONDS);
+ }
}
}
/**
*
+ * @param threadNo
*/
@SuppressWarnings({"UnusedReturnValue", "WeakerAccess"})
- @MonitoredTask(name = "Run Named Scheduled Tasks")
- protected String checkNamedTasks() {
+ @MonitoredTask(name = "Scheduled, runner", nameExtArgIndex = 0)
+ protected String checkNamedTasks(int threadNo) {
AtomicInteger run = new AtomicInteger();
List<Throwable> problems = new ArrayList<>();
namedTasks.forEach((s, task) -> {
@@ -73,7 +77,8 @@ class TcBotScheduler implements IScheduler {
problems.add(e);
}
});
- return "Finished " + run.get() + (problems.isEmpty() ? ", exceptions: " : problems.toString());
+
+ return "Finished " + run.get() + " task(s) " + (problems.isEmpty() ? "" : (", exceptions: " + problems.toString()));
}
/** {@inheritDoc} */
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
index f6f7c44..d11f148 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
@@ -72,12 +72,14 @@ public class BuildRefDao {
*/
@NotNull protected Stream<BuildRefCompacted> compactedBuildsForServer(int srvId) {
return StreamSupport.stream(buildRefsCache.spliterator(), false)
- .filter(entry -> {
- return isKeyForServer(entry.getKey(), srvId);
- })
+ .filter(entry -> isKeyForServer(entry.getKey(), srvId))
.map(javax.cache.Cache.Entry::getValue);
}
+ /**
+ * @param key Key.
+ * @param srvId Server id.
+ */
private boolean isKeyForServer(Long key, int srvId) {
return key!=null && key >> 32 == srvId;
}
@@ -86,6 +88,7 @@ public class BuildRefDao {
* @param srvId Server id mask high.
* @param ghData Gh data.
*/
+ @AutoProfiling
public Set<Long> saveChunk(long srvId, List<BuildRef> ghData) {
Set<Long> ids = ghData.stream().map(BuildRef::getId)
.filter(Objects::nonNull)
@@ -132,6 +135,7 @@ public class BuildRefDao {
* @param buildTypeId Build type id.
* @param bracnhNameQry Bracnh name query.
*/
+ @AutoProfiling
@NotNull public List<BuildRef> findBuildsInHistory(int srvId,
@Nullable String buildTypeId,
String bracnhNameQry) {
@@ -206,6 +210,7 @@ public class BuildRefDao {
* @param srvId Server id.
* @param refCompacted Reference compacted.
*/
+ @AutoProfiling
public boolean save(int srvId, BuildRefCompacted refCompacted) {
long cacheKey = buildIdToCacheKey(srvId, refCompacted.id());
BuildRefCompacted buildPersisted = buildRefsCache.get(cacheKey);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 840178e..bcd1476 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.OptionalInt;
import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -46,6 +47,7 @@ import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.ci.teamcity.pure.ITeamcityConn;
import org.apache.ignite.ci.util.ExceptionUtil;
@@ -97,11 +99,17 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
public void scheduleBuildsLoad(List<Integer> buildsToAskFromTc) {
+ if (buildsToAskFromTc.isEmpty())
+ return;
+
synchronized (this) {
buildToLoad.addAll(buildsToAskFromTc);
}
- scheduler.sheduleNamed(taskName("loadFatBuilds"), this::loadFatBuilds, 2, TimeUnit.MINUTES);
+ int ldrToActivate = ThreadLocalRandom.current().nextInt(5);
+
+ scheduler.sheduleNamed(taskName("loadFatBuilds" + ldrToActivate), () -> loadFatBuilds(ldrToActivate), 2, TimeUnit.MINUTES);
+
}
@NotNull public String taskName(String taskName) {
@@ -235,7 +243,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
runActualizeBuilds(srvId, false, collect);
// schedule full resync later
- scheduler.invokeLater(this::sheduleResync, 20, TimeUnit.MINUTES);
+ scheduler.invokeLater(this::sheduleResync, 15, TimeUnit.MINUTES);
}
/**
@@ -302,18 +310,24 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
return cacheKeysUpdated.stream().map(BuildRefDao::cacheKeyToBuildId).collect(Collectors.toList());
}
- private void loadFatBuilds() {
+ /** */
+ private void loadFatBuilds(int ldrNo) {
Set<Integer> load;
+
synchronized (this) {
load = buildToLoad;
buildToLoad = new HashSet<>();
}
- doLoadBuilds(srvId, load);
+
+ doLoadBuilds(ldrNo, srvId, load);
}
- @MonitoredTask(name = "Proactive Builds Loading", nameExtArgIndex = 0)
+ @MonitoredTask(name = "Proactive Builds Loading (agent,server)", nameExtArgsIndexes = {0, 1})
@AutoProfiling
- protected String doLoadBuilds(String srvId, Set<Integer> load) {
+ protected String doLoadBuilds(int ldrNo, String srvId, Set<Integer> load) {
+ if(load.isEmpty())
+ return "Nothing to load";
+
AtomicInteger err = new AtomicInteger();
AtomicInteger ld = new AtomicInteger();
@@ -335,6 +349,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
}
);
- return "Builds Loaded " + ld.get() + " from " + load.size() + " requested, errors: " + err;
+
+ return "Builds updated " + ld.get() + " from " + load.size() + " requested, errors: " + err;
}
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
index 6ef440b..e397d8f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedModule.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ci.teamcity.ignited;
import com.google.inject.AbstractModule;
import com.google.inject.internal.SingletonScope;
import org.apache.ignite.ci.tcbot.condition.BuildConditionDao;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
import org.apache.ignite.ci.teamcity.pure.ITeamcityHttpConnection;
import org.apache.ignite.ci.teamcity.restcached.TcRestCachedModule;
import org.jetbrains.annotations.Nullable;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/FatBuildDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
similarity index 96%
rename from ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/FatBuildDao.java
rename to ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index 43a1573..fc8b993 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/FatBuildDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.ignite.ci.teamcity.ignited;
+package org.apache.ignite.ci.teamcity.ignited.fatbuild;
import java.util.Collection;
import java.util.List;
@@ -30,6 +30,7 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.ci.db.TcHelperDb;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
+import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.jetbrains.annotations.Nullable;
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index aa2e7eb..9672abf 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -42,6 +42,7 @@ import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
import org.apache.ignite.ci.teamcity.pure.BuildHistoryEmulator;
import org.apache.ignite.ci.teamcity.pure.ITeamcityHttpConnection;
import org.apache.ignite.ci.user.ICredentialsProv;