You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sk...@apache.org on 2021/01/12 20:54:23 UTC
[ignite-teamcity-bot] branch master updated: Removed tests from
other branches in Visa's new tests list. Fixes #183
This is an automated email from the ASF dual-hosted git repository.
sk0x50 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 728d98c Removed tests from other branches in Visa's new tests list. Fixes #183
728d98c is described below
commit 728d98c2112479a171e9e104f55dd61598f88125
Author: sergeyuttsel <ut...@gmail.com>
AuthorDate: Tue Jan 12 23:53:53 2021 +0300
Removed tests from other branches in Visa's new tests list. Fixes #183
Signed-off-by: Slava Koptilin <sl...@gmail.com>
---
.../ci/tcbot/chain/MockBasedTcBotModule.java | 11 ++-
.../ignite/tcbot/engine/TcBotEngineModule.java | 5 +-
.../ignite/tcbot/engine/cleaner/Cleaner.java | 6 +-
.../ignite/tcbot/engine/newtests/NewTestInfo.java | 43 +++++++++++
.../ignite/tcbot/engine/newtests/NewTestKey.java | 36 +++++++++
.../tcbot/engine/newtests/NewTestsStorage.java | 86 ++++++++++++++++++++++
.../ignite/tcbot/engine/pr/PrChainsProcessor.java | 21 ++++--
.../apache/ignite/tcbot/engine/ui/DsChainUi.java | 20 ++++-
8 files changed, 211 insertions(+), 17 deletions(-)
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java
index daab1cf..7f778bb 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java
@@ -19,9 +19,7 @@ package org.apache.ignite.ci.tcbot.chain;
import com.google.inject.AbstractModule;
import com.google.inject.internal.SingletonScope;
-import java.io.File;
-import java.util.Properties;
-import org.apache.ignite.ci.HelperConfig;
+import org.apache.ignite.Ignite;
import org.apache.ignite.ci.github.PullRequest;
import org.apache.ignite.githubignited.IGitHubConnIgnited;
import org.apache.ignite.githubignited.IGitHubConnIgnitedProvider;
@@ -42,6 +40,7 @@ import org.apache.ignite.tcbot.engine.conf.ITrackedBranchesConfig;
import org.apache.ignite.tcbot.engine.conf.NotificationsConfig;
import org.apache.ignite.tcbot.engine.conf.TcServerConfig;
import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
+import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage;
import org.apache.ignite.tcbot.engine.user.IUserStorage;
import org.apache.ignite.tcbot.notify.IEmailSender;
import org.apache.ignite.tcbot.notify.ISlackSender;
@@ -50,12 +49,9 @@ import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
import org.apache.ignite.tcbot.persistence.InMemoryStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.TeamcityIgnitedProviderMock;
import org.apache.ignite.tcbot.common.conf.IDataSourcesConfigSupplier;
-import org.apache.ignite.tcbot.common.conf.TcBotWorkDir;
import org.apache.ignite.tcignited.buildlog.IBuildLogProcessor;
-import org.apache.ignite.tcservice.ITeamcityConn;
import org.mockito.Mockito;
-import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
@@ -140,6 +136,9 @@ public class MockBasedTcBotModule extends AbstractModule {
bind(IEmailSender.class).toInstance(Mockito.mock(IEmailSender.class));
bind(ISlackSender.class).toInstance(Mockito.mock(ISlackSender.class));
+ bind(Ignite.class).toInstance(Mockito.mock(Ignite.class));
+ bind(NewTestsStorage.class).toInstance(Mockito.mock(NewTestsStorage.class));
+
super.configure();
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java
index 515e8e6..657de21 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java
@@ -26,6 +26,7 @@ import org.apache.ignite.tcbot.engine.buildtime.BuildTimeService;
import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
import org.apache.ignite.tcbot.engine.issue.IssuesStorage;
+import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage;
import org.apache.ignite.tcbot.engine.tracked.IDetailedStatusForTrackedBranch;
import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor;
import org.apache.ignite.tcbot.engine.user.IUserStorage;
@@ -49,6 +50,8 @@ public class TcBotEngineModule extends AbstractModule {
bind(IUserStorage.class).to(UserAndSessionsStorage.class).in(new SingletonScope());
+ bind(NewTestsStorage.class).in(new SingletonScope());
+
install(new TcBotCommonModule());
}
-}
\ No newline at end of file
+}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
index dbe5499..387293c 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
@@ -26,7 +26,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
import javax.inject.Inject;
import org.apache.ignite.ci.teamcity.ignited.buildcondition.BuildConditionDao;
import org.apache.ignite.lang.IgniteBiTuple;
@@ -36,6 +35,7 @@ import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.defect.DefectsStorage;
import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
+import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.buildlog.BuildLogCheckResultDao;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
@@ -48,7 +48,6 @@ import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.mapping;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
-import static org.apache.ignite.tcignited.build.FatBuildDao.cacheKeyToSrvIdAndBuildId;
public class Cleaner {
private final AtomicBoolean init = new AtomicBoolean();
@@ -61,6 +60,7 @@ public class Cleaner {
@Inject private BuildStartTimeStorage buildStartTimeStorage;
@Inject private BuildConditionDao buildConditionDao;
@Inject private DefectsStorage defectsStorage;
+ @Inject private NewTestsStorage newTestsStorage;
@Inject private ITcBotConfig cfg;
/** Logger. */
@@ -132,6 +132,8 @@ public class Cleaner {
defectsStorage.removeOldDefects(thresholdDate.minusDays(60).toInstant().toEpochMilli(), numOfItemsToDel);
issuesStorage.removeOldIssues(thresholdDate.minusDays(60).toInstant().toEpochMilli(), numOfItemsToDel);
+ newTestsStorage.removeOldTests(ZonedDateTime.now().minusDays(5).toInstant().toEpochMilli());
+
return oldBuildsKeys.size();
}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/newtests/NewTestInfo.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/newtests/NewTestInfo.java
new file mode 100644
index 0000000..6e24814
--- /dev/null
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/newtests/NewTestInfo.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tcbot.engine.newtests;
+
+/** */
+public class NewTestInfo {
+ /** */
+ private String branch;
+
+ /** */
+ private long timestamp;
+
+ /** */
+ public NewTestInfo(String branch, long timestamp) {
+ this.branch = branch;
+ this.timestamp = timestamp;
+ }
+
+ /** */
+ public String branch() {
+ return branch;
+ }
+
+ /** */
+ public long timestamp() {
+ return timestamp;
+ }
+}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/newtests/NewTestKey.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/newtests/NewTestKey.java
new file mode 100644
index 0000000..689f441
--- /dev/null
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/newtests/NewTestKey.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tcbot.engine.newtests;
+
+/** */
+public class NewTestKey {
+ /** */
+ String srvId;
+
+ /** */
+ Long testId;
+
+ /** */
+ String baseBranch;
+
+ public NewTestKey(String srvId, Long testId, String baseBranch) {
+ this.srvId = srvId;
+ this.testId = testId;
+ this.baseBranch = baseBranch;
+ }
+}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/newtests/NewTestsStorage.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/newtests/NewTestsStorage.java
new file mode 100644
index 0000000..5bbe77a
--- /dev/null
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/newtests/NewTestsStorage.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tcbot.engine.newtests;
+
+import javax.cache.Cache;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.tcbot.persistence.CacheConfigs;
+
+/**
+ * The storage contains tests which were identified as new tests in the tcbot visa
+ */
+public class NewTestsStorage {
+ /** */
+ @Inject
+ private Provider<Ignite> igniteProvider;
+
+ /** */
+ private IgniteCache<NewTestKey, NewTestInfo> cache() {
+ return botNewTestsCache(getIgnite());
+ }
+
+ /** */
+ private Ignite getIgnite() {
+ return igniteProvider.get();
+ }
+
+ /** */
+ public static IgniteCache<NewTestKey, NewTestInfo> botNewTestsCache(Ignite ignite) {
+ CacheConfiguration<NewTestKey, NewTestInfo> ccfg = CacheConfigs.getCache8PartsConfig("newTestsCache");
+
+ return ignite.getOrCreateCache(ccfg);
+ }
+
+ /** */
+ public boolean isNewTest(String srvId, Long testId, String baseBranch, String branch) {
+ NewTestInfo savedTest = cache().get(new NewTestKey(srvId, testId, baseBranch));
+
+ if (savedTest == null)
+ return true;
+ else
+ return savedTest.branch().startsWith(branch);
+ }
+
+ /** */
+ public boolean isNewTestAndPut(String srvId, Long testId, String baseBranch, String branch) {
+ NewTestKey testKey = new NewTestKey(srvId, testId, baseBranch);
+
+ NewTestInfo savedTest = cache().get(testKey);
+
+ if (savedTest == null) {
+ cache().put(testKey, new NewTestInfo(branch, System.currentTimeMillis()));
+ return true;
+ }
+ else
+ return savedTest.branch().startsWith(branch);
+ }
+
+ /** */
+ public void removeOldTests(long thresholdDate) {
+ ScanQuery<NewTestKey, NewTestInfo> scan =
+ new ScanQuery<>((key, testInfo) -> testInfo.timestamp() < thresholdDate);
+
+ for (Cache.Entry<NewTestKey, NewTestInfo> entry : cache().query(scan))
+ cache().remove(entry.getKey());
+ }
+}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
index 8f0d56b..5f197e0 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java
@@ -17,6 +17,7 @@
package org.apache.ignite.tcbot.engine.pr;
import com.google.common.base.Strings;
+import java.time.LocalDate;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -27,7 +28,6 @@ import java.util.function.Predicate;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import javax.inject.Inject;
-import javax.inject.Provider;
import org.apache.ignite.ci.github.PullRequest;
import org.apache.ignite.githubignited.IGitHubConnIgnited;
import org.apache.ignite.githubignited.IGitHubConnIgnitedProvider;
@@ -44,6 +44,7 @@ import org.apache.ignite.tcbot.engine.chain.ProcessLogsMode;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.conf.ITrackedBranch;
import org.apache.ignite.tcbot.engine.conf.ITrackedChain;
+import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage;
import org.apache.ignite.tcbot.engine.ui.DsChainUi;
import org.apache.ignite.tcbot.engine.ui.DsSummaryUi;
import org.apache.ignite.tcbot.engine.ui.ShortSuiteUi;
@@ -59,8 +60,6 @@ import org.apache.ignite.tcignited.buildref.BranchEquivalence;
import org.apache.ignite.tcignited.creds.ICredentialsProv;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcservice.ITeamcity;
-import org.apache.ignite.tcservice.ITeamcityConn;
-import org.apache.ignite.tcservice.TeamcityServiceConnection;
/**
* Process pull request/untracked branch chain at particular server.
@@ -95,6 +94,8 @@ public class PrChainsProcessor {
@Inject private UpdateCountersStorage countersStorage;
+ @Inject private NewTestsStorage newTestsStorage;
+
/**
* @param creds Credentials.
* @param srvCodeOrAlias Server code or alias.
@@ -174,7 +175,7 @@ public class PrChainsProcessor {
//fail rate reference is always default (master)
chainStatus.initFromContext(tcIgnited, ctx, baseBranchForTc, compactor, false,
null, null, -1, null, false, false); // don't need for PR
- chainStatus.findNewTests(ctx, tcIgnited, baseBranchForTc, compactor);
+ chainStatus.findNewTests(ctx, tcIgnited, baseBranchForTc, compactor, newTestsStorage);
initJiraAndGitInfo(chainStatus, jiraIntegration, gitHubConnIgnited);
}
@@ -393,7 +394,17 @@ public class PrChainsProcessor {
.map((ctx) -> {
List<ShortTestUi> missingTests = ctx.getFilteredTests(test -> {
IRunHistory history = test.history(tcIgnited, baseBranchId, null);
- return history == null && !test.isMutedOrIgored();
+ if (history == null && !test.isMutedOrIgored()) {
+
+ if (test.getId() != null &&
+ newTestsStorage.isNewTestAndPut(tcIgnited.serverCode(),
+ test.getId(), normalizedBaseBranch, ctx.branchName()))
+ return true;
+ else
+ return false;
+ }
+ else
+ return false;
})
.stream()
.map(occurrence -> new ShortTestUi().initFrom(occurrence, occurrence.isPassed()))
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java
index f31aafe..165e97d 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsChainUi.java
@@ -18,6 +18,7 @@
package org.apache.ignite.tcbot.engine.ui;
import com.google.common.base.Strings;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
@@ -32,11 +33,11 @@ import org.apache.ignite.tcbot.common.util.CollectionUtil;
import org.apache.ignite.tcbot.engine.chain.FullChainRunCtx;
import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx;
import org.apache.ignite.tcbot.engine.chain.TestCompactedMult;
+import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage;
import org.apache.ignite.tcbot.engine.tracked.DisplayMode;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.history.IRunHistory;
-import org.apache.ignite.tcservice.ITeamcityConn;
import org.apache.ignite.tcservice.model.conf.BuildType;
import static org.apache.ignite.tcbot.engine.ui.DsSuiteUi.createOccurForLogConsumer;
@@ -294,15 +295,28 @@ public class DsChainUi {
public void findNewTests(FullChainRunCtx ctx,
ITeamcityIgnited tcIgnited,
String baseBranchTc,
- IStringCompactor compactor){
+ IStringCompactor compactor,
+ NewTestsStorage newTestsStorage){
String failRateNormalizedBranch = normalizeBranch(baseBranchTc);
Integer baseBranchId = compactor.getStringIdIfPresent(failRateNormalizedBranch);
+
newTestsUi = ctx
.suites()
.map((suite) -> {
List<ShortTestUi> missingTests = suite.getFilteredTests(test -> {
IRunHistory history = test.history(tcIgnited, baseBranchId, null);
- return history == null && !test.isMutedOrIgored();
+
+ if (history == null && !test.isMutedOrIgored()) {
+
+ if (test.getId() != null &&
+ newTestsStorage.isNewTest(tcIgnited.serverCode(),
+ test.getId(), failRateNormalizedBranch, ctx.branchName()))
+ return true;
+ else
+ return false;
+ }
+ else
+ return false;
})
.stream()
.map(occurrence -> {