You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/10/06 14:24:27 UTC
[ignite-teamcity-bot] branch ignite-9800 updated: IGNITE-9800:
paged uploading of data
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9800
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9800 by this push:
new c87ef2e IGNITE-9800: paged uploading of data
c87ef2e is described below
commit c87ef2e447025dfdb44adb96e33390f61c627513
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Sat Oct 6 17:24:24 2018 +0300
IGNITE-9800: paged uploading of data
---
.../ci/github/ignited/GitHubConnIgnitedImpl.java | 13 +++--
.../ci/github/pure/GitHubConnectionImpl.java | 62 +++++++++++++++++++---
.../ignite/ci/github/pure/IGitHubConnection.java | 4 +-
.../tcbot/visa/TcBotTriggerAndSignOffService.java | 2 +-
.../java/org/apache/ignite/ci/util/HttpUtil.java | 19 ++++++-
.../GitHubPrsParseTest.java} | 21 ++++++--
6 files changed, 103 insertions(+), 18 deletions(-)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
index ff95b68..458462d 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import javax.inject.Inject;
@@ -81,7 +82,7 @@ class GitHubConnIgnitedImpl implements IGitHubConnIgnited {
@AutoProfiling
@Override public List<PullRequest> getPullRequests() {
scheduler.sheduleNamed(IGitHubConnIgnited.class.getSimpleName() + ".actualizePrs",
- this::actualizePrs, 2, TimeUnit.MINUTES);
+ this::actualizePrs, 4, TimeUnit.MINUTES);
return StreamSupport.stream(prCache.spliterator(), false)
.filter(entry -> entry.getKey() >> 32 == srvIdMaskHigh)
@@ -107,9 +108,15 @@ class GitHubConnIgnitedImpl implements IGitHubConnIgnited {
@MonitoredTask(name = "Actualize PRs", nameExtArgIndex = 0)
@AutoProfiling
protected String runAtualizePrs(String srvId) {
- List<PullRequest> ghData = conn.getPullRequests();
-
+ AtomicReference<String> outLinkNext = new AtomicReference<>();
+ List<PullRequest> ghData = conn.getPullRequests(null, outLinkNext);
int size = saveChunk(ghData);
+ while (outLinkNext.get() != null) {
+ String nextPageUrl = outLinkNext.get();
+ ghData = conn.getPullRequests(nextPageUrl, outLinkNext);
+ size += saveChunk(ghData);
+ }
+
return "Entries saved " + size;
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/GitHubConnectionImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/GitHubConnectionImpl.java
index 08e6064..3c3c8d4 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/GitHubConnectionImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/GitHubConnectionImpl.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ci.github.pure;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.gson.Gson;
@@ -28,15 +29,19 @@ import java.io.InputStreamReader;
import java.io.UncheckedIOException;
import java.lang.reflect.Type;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Properties;
+import java.util.StringTokenizer;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.ignite.ci.HelperConfig;
import org.apache.ignite.ci.di.AutoProfiling;
import org.apache.ignite.ci.github.PullRequest;
import org.apache.ignite.ci.util.ExceptionUtil;
import org.apache.ignite.ci.util.HttpUtil;
+import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,6 +57,35 @@ class GitHubConnectionImpl implements IGitHubConnection {
/** GitHub authorization token. */
private String gitAuthTok;
+ @Nullable public static String parseNextLinkFromLinkRspHeader(String s) {
+ String nextLink = null;
+ StringTokenizer tokenizer = new StringTokenizer(s, ",");
+ for (; tokenizer.hasMoreTokens(); ) {
+ String tok = tokenizer.nextToken();
+
+ List<String> linkAndRel = new ArrayList<>();
+ StringTokenizer tokenizerForLink = new StringTokenizer(tok, ";");
+ for (; tokenizerForLink.hasMoreTokens(); ) {
+ String nextTok = tokenizerForLink.nextToken();
+ linkAndRel.add(nextTok);
+ }
+
+ if (linkAndRel.size() >= 2) {
+ String linkType = linkAndRel.get(1);
+ if ("rel=\"next\"".equals(linkType.trim()))
+ nextLink = linkAndRel.get(0).trim();
+ }
+ }
+
+ if (!isNullOrEmpty(nextLink)) {
+ if (nextLink.startsWith("<"))
+ nextLink = nextLink.substring(1);
+ if (nextLink.endsWith(">"))
+ nextLink = nextLink.substring(0, nextLink.length() - 1);
+ }
+ return nextLink;
+ }
+
/** {@inheritDoc} */
@Override public void init(String srvId) {
final File workDir = HelperConfig.resolveWorkDir();
@@ -67,7 +101,7 @@ class GitHubConnectionImpl implements IGitHubConnection {
/** {@inheritDoc} */
@AutoProfiling
@Override public PullRequest getPullRequest(String branchForTc) {
- Preconditions.checkState( !isNullOrEmpty(gitApiUrl) , "Git API URL is not configured for this server.");
+ Preconditions.checkState(!isNullOrEmpty(gitApiUrl), "Git API URL is not configured for this server.");
String id = null;
@@ -84,7 +118,7 @@ class GitHubConnectionImpl implements IGitHubConnection {
String pr = gitApiUrl + "pulls/" + id;
- try (InputStream is = HttpUtil.sendGetToGit(gitAuthTok, pr)) {
+ try (InputStream is = HttpUtil.sendGetToGit(gitAuthTok, pr, null)) {
InputStreamReader reader = new InputStreamReader(is);
return new Gson().fromJson(reader, PullRequest.class);
@@ -103,7 +137,7 @@ class GitHubConnectionImpl implements IGitHubConnection {
return true;
}
catch (IOException e) {
- logger.error("Failed to notify Git [errMsg="+e.getMessage()+']');
+ logger.error("Failed to notify Git [errMsg=" + e.getMessage() + ']');
return false;
}
@@ -121,18 +155,32 @@ class GitHubConnectionImpl implements IGitHubConnection {
/** {@inheritDoc} */
@AutoProfiling
- @Override public List<PullRequest> getPullRequests() {
- Preconditions.checkState( !isNullOrEmpty(gitApiUrl) , "Git API URL is not configured for this server.");
+ @Override public List<PullRequest> getPullRequests(@Nullable String fullUrl,
+ @Nullable AtomicReference<String> outLinkNext) {
+ Preconditions.checkState(!isNullOrEmpty(gitApiUrl), "Git API URL is not configured for this server.");
- String s = gitApiUrl + "pulls?sort=updated&direction=desc";
+ String url = fullUrl != null ? fullUrl : gitApiUrl + "pulls?sort=updated&direction=desc";
+ HashMap<String, String> rspHeaders = new HashMap<>();
+ if (outLinkNext != null) {
+ outLinkNext.set(null);
+ rspHeaders.put("Link", null); // requesting header
+ }
- try (InputStream stream = HttpUtil.sendGetToGit(gitAuthTok, s)) {
+ try (InputStream stream = HttpUtil.sendGetToGit(gitAuthTok, url, rspHeaders)) {
InputStreamReader reader = new InputStreamReader(stream);
Type listType = new TypeToken<ArrayList<PullRequest>>() {
}.getType();
+
List<PullRequest> list = new Gson().fromJson(reader, listType);
+ String link = rspHeaders.get("Link");
+ if (link != null) {
+ String nextLink = parseNextLinkFromLinkRspHeader(link);
+ if (nextLink != null)
+ outLinkNext.set(nextLink);
+ }
+ System.err.println(link);
return list;
}
catch (IOException e) {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/IGitHubConnection.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/IGitHubConnection.java
index 18bca6b..268d2c7 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/IGitHubConnection.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/pure/IGitHubConnection.java
@@ -17,7 +17,9 @@
package org.apache.ignite.ci.github.pure;
import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.ignite.ci.github.PullRequest;
+import org.jetbrains.annotations.Nullable;
public interface IGitHubConnection {
@@ -48,5 +50,5 @@ public interface IGitHubConnection {
*/
String gitApiUrl();
- List<PullRequest> getPullRequests();
+ List<PullRequest> getPullRequests(@Nullable String fullUrl, @Nullable AtomicReference<String> outLinkNext);
}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java
index 34ccf6b..d3429e3 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java
@@ -153,7 +153,7 @@ public class TcBotTriggerAndSignOffService {
IGitHubConnection gitHubConn = gitHubConnectionProvider.server(srvId);
PullRequest pr = gitHubConn.getPullRequest(branchForTc);
- ticketId = TcBotTriggerAndSignOffService.getTicketId(pr);
+ ticketId = getTicketId(pr);
if (ticketId.isEmpty()) {
jiraRes = "JIRA ticket can't be commented - " +
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/HttpUtil.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/HttpUtil.java
index 60db238..817fa04 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/HttpUtil.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/util/HttpUtil.java
@@ -31,8 +31,10 @@ import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.ignite.ci.web.rest.exception.ServiceUnauthorizedException;
+import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,6 +47,9 @@ public class HttpUtil {
/** Logger. */
private static final Logger logger = LoggerFactory.getLogger(HttpUtil.class);
+ /**
+ * @param inputStream Input stream.
+ */
private static String readIsToString(InputStream inputStream) throws IOException {
if (inputStream == null)
return "<null>";
@@ -92,10 +97,11 @@ public class HttpUtil {
*
* @param githubAuthTok Authorization OAuth token.
* @param url URL.
+ * @param rspHeaders [IN] - required codes name->null, [OUT] required codes: name->value.
* @return Input stream from connection.
* @throws IOException If failed.
*/
- public static InputStream sendGetToGit(String githubAuthTok, String url) throws IOException {
+ public static InputStream sendGetToGit(String githubAuthTok, String url, @Nullable Map<String, String> rspHeaders) throws IOException {
Stopwatch started = Stopwatch.createStarted();
URL obj = new URL(url);
HttpURLConnection con = (HttpURLConnection)obj.openConnection();
@@ -107,8 +113,17 @@ public class HttpUtil {
con.setRequestProperty("Connection", "Keep-Alive");
con.setRequestProperty("Keep-Alive", "header");
+ int resCode = con.getResponseCode();
+
+ if(rspHeaders != null) {
+ rspHeaders.keySet().forEach((k) -> {
+ String link = con.getHeaderField(k);
+
+ rspHeaders.put(k, link);
+ });
+ }
logger.info(Thread.currentThread().getName() + ": Required: " + started.elapsed(TimeUnit.MILLISECONDS)
- + "ms : Sending 'GET' request to : " + url);
+ + "ms : Sending 'GET' request to : " + url + " Response: " + resCode);
return getInputStream(con);
}
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/github/PrsParseTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/github/pure/GitHubPrsParseTest.java
similarity index 63%
rename from ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/github/PrsParseTest.java
rename to ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/github/pure/GitHubPrsParseTest.java
index 33cc08a..adb25f7 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/github/PrsParseTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/github/pure/GitHubPrsParseTest.java
@@ -14,29 +14,42 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.ignite.ci.github;
+package org.apache.ignite.ci.github.pure;
import com.google.common.base.Preconditions;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
-import java.io.FileNotFoundException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.List;
+import org.apache.ignite.ci.github.PullRequest;
+import org.apache.ignite.ci.github.pure.GitHubConnectionImpl;
import org.junit.Test;
-public class PrsParseTest {
+import static junit.framework.TestCase.assertEquals;
+
+public class GitHubPrsParseTest {
@Test
public void parse() {
InputStream stream = this.getClass().getResourceAsStream("/prsList.json");
Preconditions.checkNotNull(stream, "Can't find resource");
- Type listType = new TypeToken<ArrayList<PullRequest>>(){}.getType();
+ Type listType = new TypeToken<ArrayList<PullRequest>>() {
+ }.getType();
List<PullRequest> list = new Gson().fromJson(new InputStreamReader(stream), listType);
System.out.println(list.size());
System.out.println(list);
}
+
+ @Test
+ public void parseLinkRspHeader() {
+ String s = "<https://api.github.com/repositories/31006158/pulls?sort=updated&direction=desc&page=2>; rel=\"next\", <https://api.github.com/repositories/31006158/pulls?sort=updated&direction=desc&page=45>; rel=\"last\"\n";
+ String nextLink = GitHubConnectionImpl.parseNextLinkFromLinkRspHeader(s);
+
+ assertEquals("https://api.github.com/repositories/31006158/pulls?sort=updated&direction=desc&page=2", nextLink);
+ }
+
}