You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/12 17:16:17 UTC
[ignite-teamcity-bot] branch ignite-9848-load-all-builds updated
(324785c -> 1daf2ff)
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a change to branch ignite-9848-load-all-builds
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git.
from 324785c IGNITE-9848: Compressed fat build development
add 84bbae6 IGNITE-9869: Bug fix for Running builds not uploaded into prefetched builds collection: fixed JAXB 2 roots problem - Fixes #37.
new 9b6745b Merge branch 'master' into ignite-9848-load-all-builds
new 1daf2ff IGNITE-9869: After merge fixes
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../ignite/ci/di/scheduler/NoOpSheduler.java} | 6 +-
.../apache/ignite/ci/tcmodel/hist/BuildRef.java | 3 +-
.../org/apache/ignite/ci/tcmodel/hist/Builds.java | 15 +++
.../ci/teamcity/ignited/BuildRefCompacted.java | 5 +
.../ignite/ci/teamcity/ignited/BuildRefDao.java | 42 +++++---
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 45 +++++----
.../java/org/apache/ignite/ci/util/XmlUtil.java | 15 +--
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 85 +++++++++++++++-
.../ci/teamcity/pure/BuildHistoryEmulator.java | 108 +++++++++++++++++++++
10 files changed, 282 insertions(+), 44 deletions(-)
copy ignite-tc-helper-web/src/{test/java/org/apache/ignite/ci/di/scheduler/DirectExecNoWaitSheduler.java => main/java/org/apache/ignite/ci/di/scheduler/NoOpSheduler.java} (92%)
create mode 100644 ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/pure/BuildHistoryEmulator.java
[ignite-teamcity-bot] 01/02: Merge branch 'master' into
ignite-9848-load-all-builds
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-load-all-builds
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 9b6745b6527fbc8fe84421074f63aeaac36ccc20
Merge: 324785c 84bbae6
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Oct 12 20:13:24 2018 +0300
Merge branch 'master' into ignite-9848-load-all-builds
# Conflicts:
# ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
# ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
# ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
# ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/XmlUtil.java
# ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
.../Builds.java => di/scheduler/NoOpSheduler.java} | 34 +++----
.../apache/ignite/ci/tcmodel/hist/BuildRef.java | 3 +-
.../org/apache/ignite/ci/tcmodel/hist/Builds.java | 15 +++
.../ignite/ci/teamcity/ignited/BuildRefDao.java | 42 +++++---
.../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 45 +++++----
.../java/org/apache/ignite/ci/util/XmlUtil.java | 15 +--
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 41 ++++++--
.../ci/teamcity/pure/BuildHistoryEmulator.java | 108 +++++++++++++++++++++
9 files changed, 237 insertions(+), 68 deletions(-)
diff --cc ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
index 51df04f,2b68806..9f12097
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
@@@ -33,7 -33,7 +33,8 @@@ import org.apache.ignite.Ignite
import org.apache.ignite.IgniteCache;
import org.apache.ignite.ci.db.TcHelperDb;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
+import org.apache.ignite.configuration.CacheConfiguration;
+ import org.apache.ignite.internal.util.GridIntList;
import org.jetbrains.annotations.NotNull;
public class BuildRefDao {
@@@ -50,24 -50,20 +51,24 @@@
@Inject private IStringCompactor compactor;
public void init () {
- Ignite ignite = igniteProvider.get();
- buildsCache = ignite.getOrCreateCache(TcHelperDb.getCacheV2Config(TEAMCITY_BUILD_CACHE_NAME));
+ CacheConfiguration<Long, BuildRefCompacted> cfg = TcHelperDb.getCacheV2Config(TEAMCITY_BUILD_CACHE_NAME);
+ buildsCache = igniteProvider.get().getOrCreateCache(cfg);
}
- @NotNull protected Stream<BuildRefCompacted> compactedBuildsForServer(long srvIdMaskHigh) {
+ @NotNull protected Stream<BuildRefCompacted> compactedBuildsForServer(long srvId) {
return StreamSupport.stream(buildsCache.spliterator(), false)
- .filter(entry -> entry.getKey() >> 32 == srvIdMaskHigh)
+ .filter(entry -> entry.getKey() >> 32 == srvId)
.map(javax.cache.Cache.Entry::getValue);
}
+ /**
+ * @param srvIdMaskHigh Server id mask high.
+ * @param ghData Gh data.
+ */
- public int saveChunk(long srvIdMaskHigh, List<BuildRef> ghData) {
+ public int saveChunk(long srvId, List<BuildRef> ghData) {
Set<Long> ids = ghData.stream().map(BuildRef::getId)
.filter(Objects::nonNull)
- .map(buildId -> buildIdToCacheKey(srvIdMaskHigh, buildId))
+ .map(buildId -> buildIdToCacheKey(srvId, buildId))
.collect(Collectors.toSet());
Map<Long, BuildRefCompacted> existingEntries = buildsCache.getAll(ids);
@@@ -78,7 -74,7 +79,7 @@@
.collect(Collectors.toList());
for (BuildRefCompacted next : collect) {
- long cacheKey = buildIdToCacheKey(srvIdMaskHigh, next.id());
- long cacheKey = buildIdToCacheKey(srvId, next.id);
++ long cacheKey = buildIdToCacheKey(srvId, next.id());
BuildRefCompacted buildPersisted = existingEntries.get(cacheKey);
if (buildPersisted == null || !buildPersisted.equals(next))
@@@ -116,9 -112,9 +117,9 @@@
if (bracnhNameQryId == null)
return Collections.emptyList();
- return compactedBuildsForServer(srvIdMaskHigh)
+ return compactedBuildsForServer(srvId)
- .filter(e -> e.buildTypeId == (int)buildTypeIdId)
- .filter(e -> e.branchName == (int)bracnhNameQryId)
+ .filter(e -> e.buildTypeId() == buildTypeIdId)
+ .filter(e -> e.branchName() == bracnhNameQryId)
.map(compacted -> compacted.toBuildRef(compactor))
.collect(Collectors.toList());
}
diff --cc ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index 66392ec,ea7e0a9..c084446
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@@ -15,7 -15,7 +15,6 @@@
* limitations under the License.
*/
package org.apache.ignite.ci.teamcity.ignited;
--
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
@@@ -128,61 -124,93 +123,89 @@@ public class IgnitedTcInMemoryIntegrati
);
}
+ @Test
+ public void incrementalActualizationOfBuildsContainsQueued() throws IOException {
+ ITeamcityHttpConnection http = Mockito.mock(ITeamcityHttpConnection.class);
+
+ int queuedBuildIdx = 500;
+ ArrayList<BuildRef> tcBuilds = new ArrayList<>();
+ for (int i = 0; i < 1000; i++) {
+ BuildRef e = new BuildRef();
+ e.state = i >= queuedBuildIdx ?
+ (Math.random() * 2 > 1 ? BuildRef.STATE_QUEUED : BuildRef.STATE_RUNNING)
+ : BuildRef.STATE_FINISHED;
+ e.status = BuildRef.STATUS_SUCCESS;
+ e.buildTypeId = "IgniteTests24Java8_RunAll";
+ e.branchName = "refs/heads/master";
+ e.setId(i + 50000);
+ tcBuilds.add(e);
+ }
++ finally {
++ ignite.close();
++ }
+
- BuildHistoryEmulator emulator = new BuildHistoryEmulator(tcBuilds);
-
- when(http.sendGet(anyString(), anyString())).thenAnswer(
- (invocationOnMock) -> {
- String url = invocationOnMock.getArgument(1);
++ statues = hist.stream().map(BuildRef::state).distinct().collect(Collectors.toList());
+
- InputStream stream = emulator.handleUrl(url);
++ System.out.println("After " + statues);
++ }
+
- if (stream != null)
- return stream;
+
- throw new FileNotFoundException(url);
- }
- );
+ /**
+ *
+ */
+ @NotNull public ICredentialsProv creds() {
+ ICredentialsProv mock = Mockito.mock(ICredentialsProv.class);
- TeamcityIgnitedModule module = new TeamcityIgnitedModule();
+ when(mock.hasAccess(anyString())).thenReturn(true);
+ when(mock.getUser(anyString())).thenReturn("mtcga");
+ when(mock.getPassword(anyString())).thenReturn("123");
- module.overrideHttp(http);
+ return mock;
+ }
- Injector injector = Guice.createInjector(module, new AbstractModule() {
+ @Test
+ public void testFatBuild() throws JAXBException, IOException {
+ InputStream stream = getClass().getResourceAsStream("/build.xml");
+ Build refBuild = XmlUtil.load(Build.class, new InputStreamReader(stream));
+ Injector injector = Guice.createInjector(new AbstractModule() {
@Override protected void configure() {
bind(Ignite.class).toInstance(ignite);
- bind(IScheduler.class).to(NoOpSheduler.class);
+ bind(IStringCompactor.class).to(IgniteStringCompactor.class);
}
});
- ITeamcityIgnited srv = injector.getInstance(ITeamcityIgnitedProvider.class).server(APACHE, creds());
+ FatBuildDao instance = injector.getInstance(FatBuildDao.class);
+ instance.init();
- TeamcityIgnitedImpl teamcityIgnited = (TeamcityIgnitedImpl)srv;
- teamcityIgnited.fullReindex();
- String buildTypeId = "IgniteTests24Java8_RunAll";
- String branchName = "<default>";
- List<String> statues = srv.getBuildHistory(buildTypeId, branchName).stream().map(BuildRef::state).distinct().collect(Collectors.toList());
- System.out.println("Before " + statues);
-
- for (int i = queuedBuildIdx; i < tcBuilds.size(); i++)
- tcBuilds.get(i).state = BuildRef.STATE_FINISHED;
-
- teamcityIgnited.actualizeRecentBuilds();
-
-
- List<BuildRef> hist = srv.getBuildHistory(buildTypeId, branchName);
-
- assertTrue(!hist.isEmpty());
+ int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(APACHE);
+ int i = instance.saveChunk(srvIdMaskHigh, Collections.singletonList(refBuild));
+ assertEquals(1, i);
- for (BuildRef h : hist) {
- assertEquals(buildTypeId, h.suiteId());
+ FatBuildCompacted fatBuild = instance.getFatBuild(srvIdMaskHigh, 2039380);
- assertEquals("refs/heads/master", h.branchName());
+ Build actBuild = fatBuild.toBuild(injector.getInstance(IStringCompactor.class));
- assertTrue("Build " + h + " is expected to be finished" , h.isFinished());
+ try (FileWriter writer = new FileWriter("src/test/resources/build1.xml")) {
+ writer.write(XmlUtil.save(refBuild));
}
- statues = hist.stream().map(BuildRef::state).distinct().collect(Collectors.toList());
-
- System.out.println("After " + statues);
- }
-
- /**
- *
- */
- @NotNull public ICredentialsProv creds() {
- ICredentialsProv mock = Mockito.mock(ICredentialsProv.class);
-
- when(mock.hasAccess(anyString())).thenReturn(true);
- when(mock.getUser(anyString())).thenReturn("mtcga");
- when(mock.getPassword(anyString())).thenReturn("123");
+ String save = XmlUtil.save(actBuild);
+ System.out.println(save);
+ try (FileWriter writer = new FileWriter("src/test/resources/build2.xml")) {
+ writer.write(save);
+ }
- return mock;
+ assertEquals(refBuild.getId(), actBuild.getId());
+ assertEquals(refBuild.status(), actBuild.status());
+ assertEquals(refBuild.state(), actBuild.state());
+ assertEquals(refBuild.buildTypeId(), actBuild.buildTypeId());
+ assertEquals(refBuild.getStartDate(), actBuild.getStartDate());
+ assertEquals(refBuild.getFinishDate(), actBuild.getFinishDate());
+ BuildType refBt = refBuild.getBuildType();
+ BuildType actBt = actBuild.getBuildType();
+ assertEquals(refBt.getName(), actBt.getName());
+ assertEquals(refBt.getProjectId(), actBt.getProjectId());
+ assertEquals(refBt.getId(), actBt.getId());
}
+
}
[ignite-teamcity-bot] 02/02: IGNITE-9869: After merge fixes
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9848-load-all-builds
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 1daf2ffb78bf5834a620db96fad354faeccbd8a9
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Fri Oct 12 20:16:04 2018 +0300
IGNITE-9869: After merge fixes
---
.../ci/teamcity/ignited/BuildRefCompacted.java | 5 ++
.../ignite/ci/teamcity/ignited/BuildRefDao.java | 2 +-
.../ignited/IgnitedTcInMemoryIntegrationTest.java | 62 +++++++++++++++++++++-
3 files changed, 66 insertions(+), 3 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
index 3becbe4..3c455d3 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
@@ -107,4 +107,9 @@ public class BuildRefCompacted {
public int branchName() {
return branchName;
}
+
+ /** */
+ public int state() {
+ return state;
+ }
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
index 9f12097..ee97836 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefDao.java
@@ -139,7 +139,7 @@ public class BuildRefDao {
return compactedBuildsForServer(srvId)
- .filter(e -> list.contains(e.state) )
+ .filter(e -> list.contains(e.state()) )
.collect(Collectors.toList());
}
}
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index c084446..93710ac 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -19,20 +19,27 @@ import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import java.io.FileNotFoundException;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
+import javax.xml.bind.JAXBException;
import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.ci.di.scheduler.DirectExecNoWaitSheduler;
import org.apache.ignite.ci.di.scheduler.IScheduler;
import org.apache.ignite.ci.di.scheduler.NoOpSheduler;
+import org.apache.ignite.ci.tcmodel.conf.BuildType;
import org.apache.ignite.ci.tcmodel.hist.BuildRef;
+import org.apache.ignite.ci.tcmodel.result.Build;
import org.apache.ignite.ci.teamcity.pure.BuildHistoryEmulator;
import org.apache.ignite.ci.teamcity.pure.ITeamcityHttpConnection;
import org.apache.ignite.ci.user.ICredentialsProv;
+import org.apache.ignite.ci.util.XmlUtil;
import org.jetbrains.annotations.NotNull;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -123,6 +130,7 @@ public class IgnitedTcInMemoryIntegrationTest {
);
}
+
@Test
public void incrementalActualizationOfBuildsContainsQueued() throws IOException {
ITeamcityHttpConnection http = Mockito.mock(ITeamcityHttpConnection.class);
@@ -140,8 +148,58 @@ public class IgnitedTcInMemoryIntegrationTest {
e.setId(i + 50000);
tcBuilds.add(e);
}
- finally {
- ignite.close();
+
+ BuildHistoryEmulator emulator = new BuildHistoryEmulator(tcBuilds);
+
+ when(http.sendGet(anyString(), anyString())).thenAnswer(
+ (invocationOnMock) -> {
+ String url = invocationOnMock.getArgument(1);
+
+ InputStream stream = emulator.handleUrl(url);
+
+ if (stream != null)
+ return stream;
+
+ throw new FileNotFoundException(url);
+ }
+ );
+
+ TeamcityIgnitedModule module = new TeamcityIgnitedModule();
+
+ module.overrideHttp(http);
+
+ Injector injector = Guice.createInjector(module, new AbstractModule() {
+ @Override protected void configure() {
+ bind(Ignite.class).toInstance(ignite);
+ bind(IScheduler.class).to(NoOpSheduler.class);
+ }
+ });
+
+ ITeamcityIgnited srv = injector.getInstance(ITeamcityIgnitedProvider.class).server(APACHE, creds());
+
+ TeamcityIgnitedImpl teamcityIgnited = (TeamcityIgnitedImpl)srv;
+ teamcityIgnited.fullReindex();
+ String buildTypeId = "IgniteTests24Java8_RunAll";
+ String branchName = "<default>";
+ List<String> statues = srv.getBuildHistory(buildTypeId, branchName).stream().map(BuildRef::state).distinct().collect(Collectors.toList());
+ System.out.println("Before " + statues);
+
+ for (int i = queuedBuildIdx; i < tcBuilds.size(); i++)
+ tcBuilds.get(i).state = BuildRef.STATE_FINISHED;
+
+ teamcityIgnited.actualizeRecentBuilds();
+
+
+ List<BuildRef> hist = srv.getBuildHistory(buildTypeId, branchName);
+
+ assertTrue(!hist.isEmpty());
+
+ for (BuildRef h : hist) {
+ assertEquals(buildTypeId, h.suiteId());
+
+ assertEquals("refs/heads/master", h.branchName());
+
+ assertTrue("Build " + h + " is expected to be finished" , h.isFinished());
}
statues = hist.stream().map(BuildRef::state).distinct().collect(Collectors.toList());