You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/07/05 17:15:59 UTC
[ignite-teamcity-bot] 01/01: Build time collection callable: first
version, too long execution time
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch time-consumer-detection
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 01b87220905730b55b542ceba6b69bfb821f900b
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Jul 5 20:15:51 2019 +0300
Build time collection callable: first version, too long execution time
---
.../web/rest/buildtime/BuildTimeRestService.java | 58 +++++++++
ignite-tc-helper-web/src/main/webapp/index.html | 8 ++
.../tcbot/engine/buildtime/BuildTimeService.java | 48 ++++++++
.../{DsProblemRef.java => BuildTimeSummaryUi.java} | 11 +-
.../ignite/tcbot/engine/ui/DsHistoryStatUi.java | 2 +-
.../ignite/tcbot/engine/ui/DsProblemRef.java | 1 +
.../ignited/fatbuild/StatisticsCompacted.java | 2 +-
.../apache/ignite/tcignited/build/FatBuildDao.java | 137 +++++++++++++++++++++
8 files changed, 255 insertions(+), 12 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/buildtime/BuildTimeRestService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/buildtime/BuildTimeRestService.java
new file mode 100644
index 0000000..a035059
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/buildtime/BuildTimeRestService.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.web.rest.buildtime;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import org.apache.ignite.ci.user.ITcBotUserCreds;
+import org.apache.ignite.ci.web.CtxListener;
+import org.apache.ignite.tcbot.engine.buildtime.BuildTimeService;
+import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor;
+import org.apache.ignite.tcbot.engine.ui.BuildTimeSummaryUi;
+import org.apache.ignite.tcbot.engine.ui.LrTestsFullSummaryUi;
+import org.jetbrains.annotations.Nullable;
+
+@Path(BuildTimeRestService.LONG_RUNNING_SUMMARY)
+@Produces(MediaType.APPLICATION_JSON)
+public class BuildTimeRestService {
+ public static final String LONG_RUNNING_SUMMARY = "buildtime";
+
+ /** Servlet Context. */
+ @Context
+ private ServletContext ctx;
+
+ /** Current Request. */
+ @Context
+ private HttpServletRequest req;
+
+ @GET
+ @Path("analytics")
+ public BuildTimeSummaryUi loadAnalytics(@Nullable @QueryParam("branch") String branchOrNull) {
+ final ITcBotUserCreds creds = ITcBotUserCreds.get(req);
+
+ final BuildTimeService tbProc = CtxListener.getInjector(ctx).getInstance(BuildTimeService.class);
+
+ return tbProc.analytics( creds);
+ }
+}
diff --git a/ignite-tc-helper-web/src/main/webapp/index.html b/ignite-tc-helper-web/src/main/webapp/index.html
index e60ab52..6d0f5b8 100644
--- a/ignite-tc-helper-web/src/main/webapp/index.html
+++ b/ignite-tc-helper-web/src/main/webapp/index.html
@@ -37,6 +37,12 @@ function loadData() {
},
error: showErrInLoadStatus
});
+
+ $.ajax({ url: "/rest/buildtime/analytics",
+ success: function (data) {
+ $("#buildTime").html("Test");
+ $("#loadStatus").html("");
+ }, error: showErrInLoadStatus });
}
@@ -51,6 +57,8 @@ function loadData() {
<a href="index0.html"><button class="idxpgbutton"><font size="30px">λ</font><br>I like old home page</button></a>
</div>
+<div id="buildTime"></div>
+
<div id="loadStatus"></div>
<div id="version"></div>
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
new file mode 100644
index 0000000..6b33379
--- /dev/null
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tcbot.engine.buildtime;
+
+import java.util.List;
+import javax.inject.Inject;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
+import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
+import org.apache.ignite.tcbot.engine.ui.BuildTimeSummaryUi;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
+import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
+import org.apache.ignite.tcignited.build.FatBuildDao;
+import org.apache.ignite.tcignited.creds.ICredentialsProv;
+
+public class BuildTimeService {
+
+ @Inject ITeamcityIgnitedProvider tcProv;
+
+ /** Config. */
+ @Inject ITcBotConfig cfg;
+
+ @Inject FatBuildDao fatBuildDao;
+
+ public BuildTimeSummaryUi analytics(ICredentialsProv prov) {
+ String serverCode = cfg.primaryServerCode();
+
+ ITeamcityIgnited server = tcProv.server(serverCode, prov);
+
+ fatBuildDao.forEachFatBuild();
+
+ return null;
+ }
+}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeSummaryUi.java
similarity index 73%
copy from tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
copy to tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeSummaryUi.java
index 23fe3bd..7e6775f 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeSummaryUi.java
@@ -17,15 +17,6 @@
package org.apache.ignite.tcbot.engine.ui;
-/**
- * Detailed status of failures: Reference to some Issue with current suite or test detected by the Bot.
- * Currently contains only display name.
- */
-public class DsProblemRef {
- public String name;
- public String webUrl;
+public class BuildTimeSummaryUi {
- public DsProblemRef(String name) {
- this.name = name;
- }
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java
index a718e47..6107c70 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java
@@ -23,7 +23,7 @@ import javax.annotation.Nullable;
/**
* Detailed status of failures: Test/suite failure summary: contains statistic of failures and total runs for suite or for test.
*/
-public class DsHistoryStatUi {
+@SuppressWarnings("PublicField") public class DsHistoryStatUi {
/** Registered number of failures from TC helper DB */
@Nullable public Integer failures;
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
index 23fe3bd..3c98b34 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
@@ -21,6 +21,7 @@ package org.apache.ignite.tcbot.engine.ui;
* Detailed status of failures: Reference to some Issue with current suite or test detected by the Bot.
* Currently contains only display name.
*/
+@SuppressWarnings({"WeakerAccess", "PublicField"})
public class DsProblemRef {
public String name;
public String webUrl;
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
index 12a0f67..15955bf 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
@@ -153,7 +153,7 @@ public class StatisticsCompacted {
return val >= 0 ? val : null;
}
- private long findPropertyValue(int propCode) {
+ public long findPropertyValue(int propCode) {
if (keys == null)
return -1L;
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index 82da2c8..4bc074d 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -38,15 +38,22 @@ import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCompute;
import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.StatisticsCompacted;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.resources.IgniteInstanceResource;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.tcignited.history.HistoryCollector;
import org.apache.ignite.tcservice.model.changes.ChangesList;
+import org.apache.ignite.tcservice.model.hist.BuildRef;
import org.apache.ignite.tcservice.model.result.Build;
import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence;
import org.apache.ignite.tcservice.model.result.stat.Statistics;
@@ -227,6 +234,62 @@ public class FatBuildDao {
return res;
}
+ public void forEachFatBuild() {
+ IgniteCache<Long, BinaryObject> cacheBin = buildsCache.withKeepBinary();
+
+ Ignite ignite = igniteProvider.get();
+
+ IgniteCompute serversCompute = ignite.compute(ignite.cluster().forServers());
+
+ int stateRunning = compactor.getStringId(BuildRef.STATE_RUNNING);
+ Integer buildDurationId = compactor.getStringIdIfPresent(Statistics.BUILD_DURATION);
+
+ serversCompute.call(new BuiltTimeIgniteCallable(cacheBin, stateRunning, buildDurationId));
+
+
+
+ }
+
+ public static long getBuildRunningTime(int stateRunning, Integer buildDurationId,
+ BinaryObject buildBinary) {
+ Long startTs = buildBinary.field("startDate");
+
+ if (startTs == null || startTs <= 0)
+ return -1;
+
+ int buildTypeId = buildBinary.field("buildTypeId");
+ int status = buildBinary.field("status");
+ int state = buildBinary.field("state");
+
+ long runningTime = -1;
+ if(stateRunning == state)
+ runningTime = System.currentTimeMillis() - startTs;
+
+ if(runningTime<0){
+
+ if (buildDurationId != null) {
+ BinaryObject statistics = buildBinary.field("statistics");
+ if(statistics!=null) {
+ // statistics.field()
+ }
+ long val = -1; //statistics.findPropertyValue(buildDurationId);
+
+ runningTime = val >= 0 ? val : -1;
+ }
+
+
+ }
+
+ if(runningTime<0) {
+ Long finishTs= buildBinary.field("finishDate");
+
+ if(finishTs!=null)
+ runningTime = finishTs - startTs;
+ }
+
+ return runningTime;
+ }
+
private static class GetStartTimeProc implements CacheEntryProcessor<Long, BinaryObject, Long> {
public GetStartTimeProc() {
}
@@ -247,4 +310,78 @@ public class FatBuildDao {
return startDate;
}
}
+
+ public static class BuiltTimeIgniteCallable implements IgniteCallable<Long> {
+ private final IgniteCache<Long, BinaryObject> cacheBin;
+ private final int stateRunning;
+ private final Integer buildDurationId;
+ @IgniteInstanceResource
+ Ignite ignite;
+
+ public BuiltTimeIgniteCallable(IgniteCache<Long, BinaryObject> cacheBin, int stateRunning,
+ Integer buildDurationId) {
+ this.cacheBin = cacheBin;
+ this.stateRunning = stateRunning;
+ this.buildDurationId = buildDurationId;
+ }
+
+ @Override public Long call() throws Exception {
+
+ IgniteCache<Object, Object> cache = ignite.cache(TEAMCITY_FAT_BUILD_CACHE_NAME);
+
+ IgniteCache<Object, Object> cacheBin = cache.withKeepBinary();
+ QueryCursor<Cache.Entry<Long, BinaryObject>> query = cacheBin.query(
+ new ScanQuery<Long, BinaryObject>()
+ .setLocal(true));
+
+ /*.query(new SqlQuery<Long, BinaryObject>(
+ FatBuildCompacted.class,
+ " _KEY > ?")
+ .setLocal(true)
+ .setArgs(0L));*/
+
+// Iterate over the result set.
+ try (QueryCursor<Cache.Entry<Long, BinaryObject>> cursor = query) {
+
+ for (Cache.Entry<Long, BinaryObject> next : cursor) {
+
+ BinaryObject buildBinary = next.getValue();
+ long runningTime = getBuildRunningTime(stateRunning, buildDurationId, buildBinary);
+
+ System.err.println("Running " + runningTime);
+ }
+ }
+
+ /*
+ FieldsQueryCursor<List<?>> query = cacheBin.query(new SqlFieldsQuery("" +
+
+ "select startDate, buildTypeId from FatBuildCompacted where _KEY > ?")
+ .setLocal(true)
+ .setArgs(0L));
+
+// Iterate over the result set.
+ try (QueryCursor<List<?>> cursor = query) {
+ for (List<?> row : cursor)
+ System.out.println("startDate=" + row.get(0));
+
+ }
+ */
+
+ if (1 != 2)
+ return null;
+
+ Iterable<Cache.Entry<Long, BinaryObject>> entries = this.cacheBin.localEntries();
+ for (Cache.Entry<Long, BinaryObject> next : entries) {
+ Long srvAndBuild = next.getKey();
+
+ BinaryObject buildBinary = next.getValue();
+
+ Long val = getBuildRunningTime(stateRunning, buildDurationId, buildBinary);
+ if (val != null)
+ return val;
+ }
+
+ return null;
+ }
+ }
}