You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/08/06 13:04:38 UTC
[ignite-teamcity-bot] branch master updated: Additional
singlethread test for cache invalidation for history
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 5c14d9c Additional singlethread test for cache invalidation for history
5c14d9c is described below
commit 5c14d9ca0bb0ec5c3b92ed4f03fbb99659088eb0
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Tue Aug 6 16:04:30 2019 +0300
Additional singlethread test for cache invalidation for history
---
.../IgnitedTcInMemoryIntegrationTest.java | 80 +++++++++++++++++-----
.../ci/teamcity/ignited/BuildRefCompacted.java | 26 ++++++-
.../ignite/tcignited/TeamcityIgnitedImpl.java | 11 ++-
.../tcignited/buildref/BranchEquivalence.java | 11 +++
.../ignite/tcignited/buildref/BuildRefDao.java | 21 ++----
.../ignite/tcignited/history/HistoryCollector.java | 5 +-
6 files changed, 117 insertions(+), 37 deletions(-)
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcignited/IgnitedTcInMemoryIntegrationTest.java
index 03a490e..715f7d2 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/tcignited/IgnitedTcInMemoryIntegrationTest.java
@@ -21,35 +21,52 @@ import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.internal.SingletonScope;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import javax.xml.bind.JAXBException;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.ci.db.TcHelperDb;
-import org.apache.ignite.tcbot.persistence.scheduler.DirectExecNoWaitScheduler;
-import org.apache.ignite.jiraservice.IJiraIntegrationProvider;
import org.apache.ignite.ci.tcbot.chain.PrChainsProcessorTest;
-import org.apache.ignite.tcbot.engine.conf.TcBotJsonConfig;
-import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
-import org.apache.ignite.tcignited.buildlog.ILogProductSpecific;
-import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
-import org.apache.ignite.tcignited.build.FatBuildDao;
-import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
import org.apache.ignite.ci.teamcity.pure.BuildHistoryEmulator;
import org.apache.ignite.ci.user.ITcBotUserCreds;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.jiraservice.IJiraIntegrationProvider;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.common.conf.IDataSourcesConfigSupplier;
+import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
+import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
+import org.apache.ignite.tcbot.engine.conf.TcBotJsonConfig;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcbot.persistence.IgniteStringCompactor;
import org.apache.ignite.tcbot.persistence.TcBotPersistenceModule;
+import org.apache.ignite.tcbot.persistence.scheduler.DirectExecNoWaitScheduler;
import org.apache.ignite.tcbot.persistence.scheduler.IScheduler;
+import org.apache.ignite.tcignited.build.FatBuildDao;
+import org.apache.ignite.tcignited.build.ProactiveFatBuildSync;
+import org.apache.ignite.tcignited.buildlog.ILogProductSpecific;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcignited.history.BuildStartTimeStorage;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.ISuiteRunHistory;
-import org.apache.ignite.tcignited.history.BuildStartTimeStorage;
import org.apache.ignite.tcservice.ITeamcity;
import org.apache.ignite.tcservice.TeamcityServiceConnection;
import org.apache.ignite.tcservice.http.ITeamcityHttpConnection;
@@ -76,14 +93,12 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
-import javax.xml.bind.JAXBException;
-import java.io.*;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
-
import static com.google.common.base.Preconditions.checkNotNull;
-import static junit.framework.TestCase.*;
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertFalse;
+import static junit.framework.TestCase.assertNotNull;
+import static junit.framework.TestCase.assertNull;
+import static junit.framework.TestCase.assertTrue;
import static org.apache.ignite.tcbot.common.conf.TcBotWorkDir.ensureDirExist;
import static org.apache.ignite.tcbot.persistence.IgniteStringCompactor.STRINGS_CACHE;
import static org.mockito.ArgumentMatchers.anyString;
@@ -715,6 +730,37 @@ public class IgnitedTcInMemoryIntegrationTest {
return running2.stream().map(bref -> bref.toBuildRef(c)).collect(Collectors.toList());
}
+ @Test
+ public void testCachesInvalidation() {
+ TeamcityIgnitedModule module = new TeamcityIgnitedModule();
+
+ Injector injector = Guice.createInjector(module, new IgniteAndSchedulerTestModule());
+
+ IStringCompactor c = injector.getInstance(IStringCompactor.class);
+ BuildRefDao storage = injector.getInstance(BuildRefDao.class);
+ storage.init();
+
+ int srvId = ITeamcityIgnited.serverIdToInt("apache");
+ int branch = c.getStringId("myBranch");
+ AtomicInteger idGen = new AtomicInteger();
+ int buildTypeId = c.getStringId("myBuildType");
+ BuildRefCompacted ref = new BuildRefCompacted().withId(idGen.incrementAndGet()).state(0).status(2).branchName(branch).buildTypeId(buildTypeId);
+
+ storage.save(srvId, ref);
+
+ Set<Integer> branchList = Collections.singleton(branch);
+ assertEquals(1, storage.getAllBuildsCompacted(srvId, buildTypeId, branchList).size());
+
+ storage.save(srvId, ref.withId(idGen.incrementAndGet()));
+ assertEquals(2, storage.getAllBuildsCompacted(srvId, buildTypeId, branchList).size());
+
+ storage.save(srvId, ref.withId(idGen.incrementAndGet()).branchName(c.getStringId("SomeUnrelatedBranch")));
+ assertEquals(2, storage.getAllBuildsCompacted(srvId, buildTypeId, branchList).size());
+
+ storage.save(srvId, ref.withId(idGen.incrementAndGet()).branchName(branch));
+ assertEquals(3, storage.getAllBuildsCompacted(srvId, buildTypeId, branchList).size());
+ }
+
/**
*
*/
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
index b9c2d5d..50a367f 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
@@ -162,8 +162,10 @@ public class BuildRefCompacted {
}
/** */
- protected void status(int status) {
+ public BuildRefCompacted status(int status) {
this.status = status;
+
+ return this;
}
/** */
@@ -172,8 +174,10 @@ public class BuildRefCompacted {
}
/** */
- public void state(int state) {
+ public BuildRefCompacted state(int state) {
this.state = state;
+
+ return this;
}
/** */
@@ -222,4 +226,22 @@ public class BuildRefCompacted {
public String state(IStringCompactor compactor) {
return compactor.getStringFromId(state());
}
+
+ public BuildRefCompacted branchName(int branchName) {
+ this.branchName = branchName;
+
+ return this;
+ }
+
+ public BuildRefCompacted buildTypeId(int buildTypeId) {
+ this.buildTypeId = buildTypeId;
+
+ return this;
+ }
+
+ public BuildRefCompacted withId(int id) {
+ this.id = id;
+
+ return this;
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index 068a32b..e6d0778 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -361,7 +361,16 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
@Nullable String branchName) {
ensureActualizeRequested();
- return buildRefDao.getAllBuildsCompacted(srvIdMaskHigh, buildTypeId, branchEquivalence.branchForQuery(branchName));
+ Integer buildTypeIdId = compactor.getStringIdIfPresent(buildTypeId);
+ if (buildTypeIdId == null)
+ return Collections.emptyList();
+
+ Set<Integer> branchNameIds = branchEquivalence.branchIdsForQuery(branchName, compactor);
+
+ if (branchNameIds.isEmpty())
+ return Collections.emptyList();
+
+ return buildRefDao.getAllBuildsCompacted(srvIdMaskHigh, buildTypeIdId, branchNameIds);
}
/** {@inheritDoc} */
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java
index 37eb8ca..f6d7e17 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BranchEquivalence.java
@@ -19,8 +19,12 @@ package org.apache.ignite.tcignited.buildref;
import com.google.common.collect.Lists;
import java.util.Collections;
import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcservice.ITeamcity;
public class BranchEquivalence {
@@ -53,4 +57,11 @@ public class BranchEquivalence {
return Collections.singletonList(branchName);
}
+ public Set<Integer> branchIdsForQuery(@Nullable String branchName, IStringCompactor compactor) {
+ List<String> bracnhNameQry = branchForQuery(branchName);
+ return bracnhNameQry.stream()
+ .map(compactor::getStringIdIfPresent)
+ .filter(Objects::nonNull)
+ .collect(Collectors.toSet());
+ }
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
index c03f197..91c73b0 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
@@ -19,6 +19,7 @@ package org.apache.ignite.tcignited.buildref;
import com.google.common.cache.CacheBuilder;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -194,23 +195,13 @@ public class BuildRefDao {
/**
* @param srvId Server id mask high.
- * @param buildTypeId Build type (suite) id.
- * @param bracnhNameQry Bracnh name query.
+ * @param buildTypeIdId Build type (suite) id from compactor.
+ * @param branchNameIds Branch names for query.
*/
@AutoProfiling
@Nonnull public List<BuildRefCompacted> getAllBuildsCompacted(int srvId,
- String buildTypeId,
- List<String> bracnhNameQry) {
- Integer buildTypeIdId = compactor.getStringIdIfPresent(buildTypeId);
- if (buildTypeIdId == null)
- return Collections.emptyList();
-
- Set<Integer> branchNameIds = bracnhNameQry.stream().map(str -> compactor.getStringIdIfPresent(str))
- .filter(Objects::nonNull).collect(Collectors.toSet());
-
- if (branchNameIds.isEmpty())
- return Collections.emptyList();
-
+ int buildTypeIdId,
+ Collection<Integer> branchNameIds) {
List<BuildRefCompacted> res = new ArrayList<>();
branchNameIds.forEach(branchNameId -> {
@@ -226,7 +217,7 @@ public class BuildRefDao {
if (!resForBranch.isEmpty()) {
System.err.println("Branch " + compactor.getStringFromId(branchNameId)
- + " Suite " + buildTypeId
+ + " Suite " + compactor.getStringFromId(buildTypeIdId)
+ " builds " + resForBranch.size() + " ");
}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
index b74cc3c..3695cb6 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -150,8 +150,9 @@ public class HistoryCollector {
int srvId, int buildTypeId, int normalizedBaseBranch, Set<Integer> knownBuilds) {
String btId = compactor.getStringFromId(buildTypeId);
String branchId = compactor.getStringFromId(normalizedBaseBranch);
- List<BuildRefCompacted> bRefsList = buildRefDao.getAllBuildsCompacted(srvId, btId,
- branchEquivalence.branchForQuery(branchId));
+ Set<Integer> strings = branchEquivalence.branchIdsForQuery(branchId, compactor);
+ List<BuildRefCompacted> bRefsList =
+ buildRefDao.getAllBuildsCompacted(srvId, buildTypeId, strings);
long curTs = System.currentTimeMillis();
Set<Integer> buildIds = bRefsList.stream()