You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/01/18 17:46:55 UTC
[ignite-teamcity-bot] branch ignite-10989-remove-old-rest updated:
IGNITE-10989: Removing REST-caching, queued builds removed
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-10989-remove-old-rest
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-10989-remove-old-rest by this push:
new cd5d781 IGNITE-10989: Removing REST-caching, queued builds removed
cd5d781 is described below
commit cd5d7818020e6342051a8527db15ff7751b631e4
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Jan 18 20:46:54 2019 +0300
IGNITE-10989: Removing REST-caching, queued builds removed
---
.../main/java/org/apache/ignite/ci/ITeamcity.java | 4 --
.../apache/ignite/ci/IgnitePersistentTeamcity.java | 68 +-----------------
.../apache/ignite/ci/IgniteTeamcityConnection.java | 81 +---------------------
.../org/apache/ignite/ci/jobs/CheckQueueJob.java | 48 ++++++-------
.../ci/teamcity/ignited/ITeamcityIgnited.java | 8 +++
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 17 +++++
.../ignite/ci/teamcity/pure/ITeamcityConn.java | 6 +-
7 files changed, 53 insertions(+), 179 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 4ce064d..dce65df 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -52,10 +52,6 @@ public interface ITeamcity extends ITeamcityConn {
/** Temporary feature toggle: Use New run stat in PR analysis, issue detection. */
boolean NEW_RUN_STAT = true;
- /** */
- @Deprecated
- CompletableFuture<List<BuildRef>> getQueuedBuilds(@Nullable String branch);
-
@Deprecated
Build getBuild(String href);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index bfa6376..1d8357c 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -98,13 +98,7 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
@Nullable
private String serverId;
- /** cached running builds for branch. */
- private ConcurrentMap<String, Expirable<List<BuildRef>>> queuedBuilds = new ConcurrentHashMap<>();
-
- /** cached loads of queued builds for branch. */
- private ConcurrentMap<String, CompletableFuture<List<BuildRef>>> queuedBuildsFuts = new ConcurrentHashMap<>();
-
- //todo: not good code to keep it static
+ //todo: remove triggering dependency from getTrackedBranch processing, use TC Bot DB data.
@Deprecated
private static long lastTriggerMs = System.currentTimeMillis();
@@ -172,52 +166,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
return serverId;
}
- @Deprecated
- private <K, V> CompletableFuture<V> loadAsyncIfAbsentOrExpired(ConcurrentMap<K, Expirable<V>> cache,
- K key,
- ConcurrentMap<K, CompletableFuture<V>> cachedComputations,
- Function<K, CompletableFuture<V>> realLoadFunction,
- int maxAgeSecs,
- boolean alwaysProvidePersisted) {
- @Nullable final Expirable<V> persistedVal = cache.get(key);
-
- if (persistedVal != null && persistedVal.isAgeLessThanSecs(maxAgeSecs))
- return CompletableFuture.completedFuture(persistedVal.getData());
-
- AtomicReference<CompletableFuture<V>> submitRef = new AtomicReference<>();
-
- CompletableFuture<V> loadFut = cachedComputations.computeIfAbsent(key,
- k -> {
- CompletableFuture<V> future = realLoadFunction.apply(k)
- .thenApplyAsync(valueLoaded -> {
- final Expirable<V> cached = new Expirable<>(valueLoaded);
-
- ObjectInterner.internFields(cached);
-
- cache.put(k, cached);
-
- return valueLoaded;
- });
-
- submitRef.set(future);
-
- return future;
- }
- ).thenApply(res -> {
- CompletableFuture<V> f = submitRef.get();
-
- if (f != null)
- cachedComputations.remove(key, f);
-
- return res;
- });
-
- if (alwaysProvidePersisted && persistedVal != null)
- return CompletableFuture.completedFuture(persistedVal.getData());
-
- return loadFut;
- }
-
public static int getTriggerRelCacheValidSecs(int defaultSecs) {
long msSinceTrigger = System.currentTimeMillis() - lastTriggerMs;
@@ -226,20 +174,6 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITea
}
/** {@inheritDoc} */
- @Override public CompletableFuture<List<BuildRef>> getQueuedBuilds(@Nullable final String branch) {
- int defaultSecs = 60;
- int secondsUseCached = getTriggerRelCacheValidSecs(defaultSecs);
-
- return loadAsyncIfAbsentOrExpired(
- queuedBuilds,
- Strings.nullToEmpty(branch),
- queuedBuildsFuts,
- teamcity::getQueuedBuilds,
- secondsUseCached,
- secondsUseCached == defaultSecs);
- }
-
- /** {@inheritDoc} */
@Nullable
@AutoProfiling
@Override public Build getBuild(String href) {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
index 6a75fc8..af83f00 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
@@ -368,49 +368,6 @@ public class IgniteTeamcityConnection implements ITeamcity {
return XmlUtil.load(rootElem, reader);
}
- /**
- * @param buildTypeId Build type id.
- * @param branchName Branch name.
- * @param dfltFilter Default filter.
- * @param state State.
- * @param sinceDate Since date.
- * @param untilDate Until date.
- * @param sinceBuildId Since build id. Value is ignored if dates filter is present.
- */
- @Deprecated
- private List<BuildRef> getBuildHistory(@Nullable String buildTypeId,
- @Nullable String branchName,
- boolean dfltFilter,
- @Nullable String state,
- @Nullable Date sinceDate,
- @Nullable Date untilDate,
- @Nullable Integer sinceBuildId) {
- String btFilter = isNullOrEmpty(buildTypeId) ? "" : ",buildType:" + buildTypeId;
- String stateFilter = isNullOrEmpty(state) ? "" : (",state:" + state);
- String branchFilter = isNullOrEmpty(branchName) ? "" :",branch:" + branchName;
- String sinceDateFilter = sinceDate == null ? "" : ",sinceDate:" + getDateYyyyMmDdTHhMmSsZ(sinceDate);
- String untilDateFilter = untilDate == null ? "" : ",untilDate:" + getDateYyyyMmDdTHhMmSsZ(untilDate);
- String buildNoFilter = sinceBuildId != null && sinceDateFilter.isEmpty() && untilDateFilter.isEmpty()
- ? ",sinceBuild:(id:" + sinceBuildId + ")" : "";
-
- return sendGetXmlParseJaxb(host + "app/rest/latest/builds"
- + "?locator="
- + "defaultFilter:" + dfltFilter
- + btFilter
- + stateFilter
- + branchFilter
- + buildNoFilter
- + ",count:" + DEFAULT_BUILDS_COUNT
- + sinceDateFilter
- + untilDateFilter, Builds.class).getBuildsNonNull();
- }
-
- public String getDateYyyyMmDdTHhMmSsZ(Date date){
- return new SimpleDateFormat("yyyyMMdd'T'HHmmssZ")
- .format(date)
- .replace("+", "%2B");
- }
-
/** {@inheritDoc} */
@AutoProfiling
@Override public BuildTypeFull getBuildType(String buildTypeId) {
@@ -418,9 +375,9 @@ public class IgniteTeamcityConnection implements ITeamcity {
buildTypeId, BuildTypeFull.class);
}
- @Override
+ /** {@inheritDoc} */
@AutoProfiling
- public Build getBuild(String href) {
+ @Override public Build getBuild(String href) {
return getJaxbUsingHref(href, Build.class);
}
@@ -439,40 +396,6 @@ public class IgniteTeamcityConnection implements ITeamcity {
}
/** {@inheritDoc} */
- @Override
- @AutoProfiling public CompletableFuture<List<BuildRef>> getQueuedBuilds(@Nullable String branch) {
- return supplyAsync(() -> getBuildsInState(null, branch, BuildRef.STATE_QUEUED), executor);
- }
-
- @Deprecated
- private List<BuildRef> getBuildsInState(
- @Nullable final String projectId,
- @Nullable final String branch,
- @Nonnull final String state,
- @Nullable final Integer sinceBuildId) {
- List<BuildRef> finished = getBuildHistory(projectId,
- UrlUtil.escape(branch),
- false,
- state,
- null,
- null,
- sinceBuildId);
- return finished.stream().filter(BuildRef::isNotCancelled).collect(Collectors.toList());
- }
-
-
- @SuppressWarnings("WeakerAccess")
- @AutoProfiling
- @Deprecated
- protected List<BuildRef> getBuildsInState(
- @Nullable final String projectId,
- @Nullable final String branch,
- @Nonnull final String state) {
-
- return getBuildsInState(projectId, branch, state, null);
- }
-
- /** {@inheritDoc} */
@Override public String serverId() {
return tcName;
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jobs/CheckQueueJob.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jobs/CheckQueueJob.java
index b20363b..482cebe 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jobs/CheckQueueJob.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jobs/CheckQueueJob.java
@@ -26,24 +26,22 @@ import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
import javax.inject.Inject;
import jersey.repackaged.com.google.common.base.Throwables;
import org.apache.ignite.ci.HelperConfig;
-import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
-import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnitedProvider;
-import org.apache.ignite.ci.teamcity.restcached.ITcServerProvider;
import org.apache.ignite.ci.ITeamcity;
import org.apache.ignite.ci.conf.BranchTracked;
import org.apache.ignite.ci.conf.ChainAtServerTracked;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.di.MonitoredTask;
import org.apache.ignite.ci.tcmodel.agent.Agent;
-import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.tcmodel.user.User;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
+import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
+import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnitedProvider;
+import org.apache.ignite.ci.teamcity.restcached.ITcServerProvider;
import org.apache.ignite.ci.user.ICredentialsProv;
-import org.apache.ignite.ci.web.rest.parms.FullQueryParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -131,15 +129,9 @@ public class CheckQueueJob implements Runnable {
try {
checkQueue(srvId, chains);
}
- catch (RuntimeException | ExecutionException e) {
- logger.error("Unable to check queue: " + e.getMessage(), e);
-
- throw Throwables.propagate(e);
- }
- catch (InterruptedException e) {
+ catch (Exception e) {
logger.error("Unable to check queue: " + e.getMessage(), e);
- Thread.currentThread().interrupt();
throw Throwables.propagate(e);
}
}
@@ -155,7 +147,10 @@ public class CheckQueueJob implements Runnable {
@AutoProfiling
@MonitoredTask(name = "Check Server Queue", nameExtArgIndex = 0)
protected String checkQueue(String srvId,
- List<ChainAtServerTracked> chains) throws ExecutionException, InterruptedException {
+ List<ChainAtServerTracked> chains) {
+
+ ITeamcityIgnited tcIgn = tcIgnitedProv.server(srvId, creds);
+
ITeamcity teamcity = tcPureProv.server(srvId, creds);
List<Agent> agents = teamcity.agents(true, true);
@@ -177,26 +172,27 @@ public class CheckQueueJob implements Runnable {
if (free < CHECK_QUEUE_MIN_FREE_AGENTS_PERCENT)
return "Min agent percent of free agents not met:" + agentStatus;
- logger.info("There are more than half free agents (total={}, free={}).", total, total - running);
-
- List<BuildRef> builds = teamcity.getQueuedBuilds(null).get();
+ logger.info("There are more than {}% free agents (total={}, free={}).", CHECK_QUEUE_MIN_FREE_AGENTS_PERCENT,
+ total, total - running);
- String selfLogin = creds.getUser(teamcity.serverId());
+ String selfLogin = creds.getUser(srvId);
StringBuilder res = new StringBuilder();
for (ChainAtServerTracked chain : chains) {
- if(!Objects.equals(chain.serverId, teamcity.serverId()))
+ if(!Objects.equals(chain.serverId, srvId))
continue;
boolean trigger = true;
- for (BuildRef ref : builds) {
- Build build = teamcity.getBuild(ref.href);
+ List<BuildRefCompacted> buildsForBr = tcIgn.getQueuedBuildsCompacted(chain.branchForRest);
+ for (BuildRefCompacted refComp : buildsForBr) {
+ Integer buildId= refComp.getId();
+ Build build = teamcity.getBuild(buildId);
User user = build.getTriggered().getUser();
if (user == null) {
- logger.info("Unable to get username for queued build {} (type={}).", ref.getId(), ref.buildTypeId);
+ logger.info("Unable to get username for queued build {} (type={}).", buildId, build.buildTypeId);
continue;
}
@@ -204,11 +200,11 @@ public class CheckQueueJob implements Runnable {
String login = user.username;
if (selfLogin.equalsIgnoreCase(login)
- && Objects.equals(ref.branchName, chain.branchForRest)) {
+ && Objects.equals(build.branchName, chain.branchForRest)) {
final String msg
= MessageFormat.format("Queued build {0} was early triggered " +
"(user {1}, branch {2}, suite {3})." +
- " Will not startIgnite build.", ref.getId(), login, ref.branchName, ref.buildTypeId);
+ " Will not startIgnite build.", buildId, login, chain.branchForRest, build.buildTypeId);
logger.info(msg);
@@ -248,9 +244,7 @@ public class CheckQueueJob implements Runnable {
startTimes.put(chain, curr);
- ITeamcityIgnited srv = tcIgnitedProv.server(srvId, creds);
-
- srv.triggerBuild(chain.suiteId, chain.branchForRest, true, false);
+ tcIgn.triggerBuild(chain.suiteId, chain.branchForRest, true, false);
res.append(chain.branchForRest).append(" ").append(chain.suiteId).append(" triggered; ");
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
index 28b6ca4..4bd2fe6 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/ITeamcityIgnited.java
@@ -59,6 +59,14 @@ public interface ITeamcityIgnited {
@Nullable String branchName);
/**
+ * Return queued builds for branch and suite, without relation to its status.
+ *
+ * @param branchName Branch name.
+ * @return list of builds in history, includes all statuses: queued, running, etc
+ */
+ public List<BuildRefCompacted> getQueuedBuildsCompacted(String branchName);
+
+ /**
* @param projectId Project id.
* @param creds Credentials.
* @return Mutes for associated server and given project pair.
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 073f1a7..a8eb8f4 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -44,6 +44,7 @@ import org.apache.ignite.ci.di.cache.GuavaCached;
import org.apache.ignite.ci.di.scheduler.IScheduler;
import org.apache.ignite.ci.tcbot.trends.MasterTrendsService;
import org.apache.ignite.ci.tcmodel.conf.Project;
+import org.apache.ignite.ci.tcmodel.hist.BuildRef;
import org.apache.ignite.ci.tcmodel.mute.MuteInfo;
import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.teamcity.ignited.buildcondition.BuildCondition;
@@ -340,6 +341,22 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
}
/** {@inheritDoc} */
+ @AutoProfiling
+ @Override public List<BuildRefCompacted> getQueuedBuildsCompacted(
+ @Nullable String branchName) {
+ ensureActualizeRequested();
+
+ Integer stateQueuedId = compactor.getStringIdIfPresent(BuildRef.STATE_QUEUED);
+ if (stateQueuedId == null)
+ return Collections.emptyList();
+
+ List<BuildRefCompacted> builds = buildRefDao.getBuildsForBranch(srvIdMaskHigh, branchForQuery(branchName));
+
+ return builds.stream().filter(b -> b.state() == stateQueuedId).collect(Collectors.toList());
+ }
+
+
+ /** {@inheritDoc} */
@Override public Set<MuteInfo> getMutes(String projectId, ICredentialsProv creds) {
muteSync.ensureActualizeMutes(taskName("actualizeMutes"), projectId, srvIdMaskHigh, conn);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
index 4c69296..ce7f304 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/pure/ITeamcityConn.java
@@ -36,13 +36,15 @@ import org.apache.ignite.ci.tcmodel.result.stat.Statistics;
import org.apache.ignite.ci.tcmodel.result.tests.TestOccurrencesFull;
/**
- * Pure Teamcity Connection API for calling methods from REST service: https://confluence.jetbrains.com/display/TCD10/REST+API
+ * Pure Teamcity Connection API for calling methods from REST service: <br>
+ * https://confluence.jetbrains.com/display/TCD10/REST+API
*/
public interface ITeamcityConn {
/**
* @return Internal server ID as string
*/
- @Nullable public String serverId();
+ @Nullable
+ public String serverId();
/**
* @return Normalized Host address, ends with '/'.