You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2019/10/21 13:21:09 UTC
[asterixdb] branch master updated (4daaf2f -> fc1f23a)
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from 4daaf2f Merge branch 'gerrit/mad-hatter' into 'master'
new 37a2467 [NO ISSUE][TEST] Do Not Print regexjson Actual Test Result
new 1c3aa58 [NO ISSUE][TEST] Default test charsets to UTF-8 only
new d7ef925 [NO ISSUE][TEST] Fix extraction of multi-document result in test executor
new a9ab32e [NO ISSUE][NET] Exclude Warn Logs When No Buffer Factory is Used
new fc1f23a Merge branch 'gerrit/mad-hatter' into 'master'
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../asterix/test/common/ResultExtractor.java | 29 +++++++++
.../apache/asterix/test/common/TestExecutor.java | 72 +++++++++++++++++-----
.../test/server/SqlppExecutionNCServiceIT.java | 12 ++++
.../muxdemux/FullFrameChannelReadInterface.java | 2 +-
4 files changed, 99 insertions(+), 16 deletions(-)
[asterixdb] 05/05: Merge branch 'gerrit/mad-hatter' into 'master'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit fc1f23a12e04fa3d1a52507bc7f41d3fceba69ef
Merge: 4daaf2f a9ab32e
Author: Michael Blow <mb...@apache.org>
AuthorDate: Mon Oct 21 07:23:46 2019 -0400
Merge branch 'gerrit/mad-hatter' into 'master'
Change-Id: Ie4f5abbf708915146ca11f2074ba05bfae753626
.../asterix/test/common/ResultExtractor.java | 29 +++++++++
.../apache/asterix/test/common/TestExecutor.java | 72 +++++++++++++++++-----
.../test/server/SqlppExecutionNCServiceIT.java | 12 ++++
.../muxdemux/FullFrameChannelReadInterface.java | 2 +-
4 files changed, 99 insertions(+), 16 deletions(-)
[asterixdb] 04/05: [NO ISSUE][NET] Exclude Warn Logs When No Buffer
Factory is Used
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit a9ab32e3f14ebef15ee553118bd05f4fc024babe
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Fri Oct 18 22:04:05 2019 +0300
[NO ISSUE][NET] Exclude Warn Logs When No Buffer Factory is Used
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Do not warn about recycled partial frames when no buffer
factory is set in FullFrameChannelReadInterface as those
are valid cases that use a single frame to perform the
complete operation.
Change-Id: I80251c529eb15ee22d01c320883418ea80272c2c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/3824
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Till Westmann <ti...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../hyracks/net/protocols/muxdemux/FullFrameChannelReadInterface.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/FullFrameChannelReadInterface.java b/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/FullFrameChannelReadInterface.java
index 9d7f848..0ce5a87 100644
--- a/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/FullFrameChannelReadInterface.java
+++ b/hyracks-fullstack/hyracks/hyracks-net/src/main/java/org/apache/hyracks/net/protocols/muxdemux/FullFrameChannelReadInterface.java
@@ -47,7 +47,7 @@ public class FullFrameChannelReadInterface extends AbstractChannelReadInterface
credits = 0;
emptyBufferAcceptor = buffer -> {
final int delta = buffer.remaining();
- if (delta != frameSize) {
+ if (bufferFactory != null && delta != frameSize) {
LOGGER.warn("partial frame being recycled; expected size {}, actual size {}", frameSize, delta);
}
synchronized (bufferRecycleLock) {
[asterixdb] 01/05: [NO ISSUE][TEST] Do Not Print regexjson Actual
Test Result
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 37a246754baaf29922e323254f3b08445b692769
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Fri Oct 18 05:07:32 2019 +0300
[NO ISSUE][TEST] Do Not Print regexjson Actual Test Result
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Do not print actual test result when comparing the result
using regexjson.
Change-Id: Ia1381aa20d0ed47eb11ab8474c1acad1bd2988fb
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/3806
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Till Westmann <ti...@apache.org>
Reviewed-by: Michael Blow <mb...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Michael Blow <mb...@apache.org>
---
.../src/test/java/org/apache/asterix/test/common/TestExecutor.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 789cdbc..af82754 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -259,7 +259,6 @@ public class TestExecutor {
ObjectMapper OM = new ObjectMapper();
JsonNode expectedJson = OM.readTree(readerExpected);
JsonNode actualJson = OM.readTree(readerActual);
- System.out.println(OM.writeValueAsString(actualJson));
if (expectedJson == null || actualJson == null) {
throw new NullPointerException("Error parsing expected or actual result file for " + scriptFile);
}
[asterixdb] 03/05: [NO ISSUE][TEST] Fix extraction of
multi-document result in test executor
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d7ef925f3c3852bce601903fa7bc6068bf4cc6fa
Author: sandeepgupta <sa...@couchbase.com>
AuthorDate: Tue Oct 8 11:56:35 2019 +0530
[NO ISSUE][TEST] Fix extraction of multi-document result in test executor
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Fix extraction/writing within test suite for multiple document result as proper JSON document
Change-Id: I879e045e2ae7f64f662993afb46cad7b29c781b3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/3623
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
Tested-by: Till Westmann <ti...@apache.org>
---
.../asterix/test/common/ResultExtractor.java | 29 ++++++++++++
.../apache/asterix/test/common/TestExecutor.java | 54 ++++++++++++++++++++--
2 files changed, 78 insertions(+), 5 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
index 60d44a1..5e53d54 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
@@ -86,6 +86,12 @@ public class ResultExtractor {
private static final Logger LOGGER = LogManager.getLogger();
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ public static ExtractedResult extract(InputStream resultStream, Charset resultCharset, String outputFormat)
+ throws Exception {
+ return extract(resultStream, EnumSet.of(ResultField.RESULTS, ResultField.WARNINGS), resultCharset,
+ outputFormat);
+ }
+
public static ExtractedResult extract(InputStream resultStream, Charset resultCharset) throws Exception {
return extract(resultStream, EnumSet.of(ResultField.RESULTS, ResultField.WARNINGS), resultCharset);
}
@@ -120,6 +126,25 @@ public class ResultExtractor {
private static ExtractedResult extract(InputStream resultStream, EnumSet<ResultField> resultFields,
Charset resultCharset) throws Exception {
+ return extract(resultStream, resultFields, resultCharset, "jsonl"); //default output format type is jsonl
+ }
+
+ private static ExtractedResult extract(InputStream resultStream, EnumSet<ResultField> resultFields,
+ Charset resultCharset, String fmt) throws Exception {
+
+ if (fmt.equals("json")) {
+ return extract(resultStream, resultFields, resultCharset, "[", ",", "]");
+ }
+
+ if (fmt.equals("jsonl")) {
+ return extract(resultStream, resultFields, resultCharset, "", "", "");
+ }
+
+ throw new AsterixException("Unkown output format for result of test query");
+ }
+
+ private static ExtractedResult extract(InputStream resultStream, EnumSet<ResultField> resultFields,
+ Charset resultCharset, String openMarker, String separator, String closeMarker) throws Exception {
ExtractedResult extractedResult = new ExtractedResult();
final String resultStr = IOUtils.toString(resultStream, resultCharset);
final ObjectNode result = OBJECT_MAPPER.readValue(resultStr, ObjectNode.class);
@@ -158,7 +183,10 @@ public class ResultExtractor {
} else {
JsonNode[] fields = Iterators.toArray(fieldValue.elements(), JsonNode.class);
if (fields.length > 1) {
+ String sep = openMarker;
for (JsonNode f : fields) {
+ resultBuilder.append(sep);
+ sep = separator;
if (f.isObject()) {
resultBuilder.append(OBJECT_MAPPER.writeValueAsString(f));
@@ -166,6 +194,7 @@ public class ResultExtractor {
resultBuilder.append(f.asText());
}
}
+ resultBuilder.append(closeMarker);
}
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 15711d0..8dc04eb 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -138,6 +138,9 @@ public class TestExecutor {
Pattern.compile("polltimeoutsecs=(\\d+)(\\D|$)", Pattern.MULTILINE);
private static final Pattern POLL_DELAY_PATTERN = Pattern.compile("polldelaysecs=(\\d+)(\\D|$)", Pattern.MULTILINE);
private static final Pattern HANDLE_VARIABLE_PATTERN = Pattern.compile("handlevariable=(\\w+)");
+ private static final Pattern RESULT_VARIABLE_PATTERN = Pattern.compile("resultvariable=(\\w+)");
+ private static final Pattern OUTPUTFORMAT_VARIABLE_PATTERN = Pattern.compile("outputformat=(\\w+)");
+
private static final Pattern VARIABLE_REF_PATTERN = Pattern.compile("\\$(\\w+)");
private static final Pattern HTTP_PARAM_PATTERN =
Pattern.compile("param (?<name>[\\w-$]+)(?::(?<type>\\w+))?=(?<value>.*)", Pattern.MULTILINE);
@@ -236,6 +239,9 @@ public class TestExecutor {
ComparisonEnum compare, Charset actualEncoding) throws Exception {
LOGGER.info("Expected results file: {} ", expectedFile);
boolean regex = false;
+ if (expectedFile.getName().endsWith(".ignore")) {
+ return; //skip the comparison
+ }
try (BufferedReader readerExpected =
new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), UTF_8));
BufferedReader readerActual =
@@ -1318,13 +1324,14 @@ public class TestExecutor {
}
boolean isJsonEncoded = isJsonEncoded(extractHttpRequestType(statement));
-
Charset responseCharset = getResponseCharset(expectedResultFile);
InputStream resultStream;
ExtractedResult extractedResult = null;
+ final String variablesReplaced = replaceVarRefRelaxed(statement, variableCtx);
+ String resultVar = getResultVariable(statement); //Is the result of the statement/query to be used in later tests
if (DELIVERY_IMMEDIATE.equals(delivery)) {
- resultStream = executeQueryService(statement, ctx, fmt, uri, params, isJsonEncoded, responseCharset, null,
- isCancellable(reqType));
+ resultStream = executeQueryService(variablesReplaced, ctx, fmt, uri, params, isJsonEncoded, responseCharset,
+ null, isCancellable(reqType));
switch (reqType) {
case METRICS_QUERY_TYPE:
resultStream = ResultExtractor.extractMetrics(resultStream, responseCharset);
@@ -1336,7 +1343,13 @@ public class TestExecutor {
resultStream = ResultExtractor.extractPlans(resultStream, responseCharset);
break;
default:
- extractedResult = ResultExtractor.extract(resultStream, responseCharset);
+ String outputFormatVariable = getOutputFormatVariable(statement);
+ if ((outputFormatVariable == null) || (outputFormatVariable.equals("jsonl"))) {
+ extractedResult = ResultExtractor.extract(resultStream, responseCharset);
+ } else {
+ extractedResult = ResultExtractor.extract(resultStream, responseCharset, "json");
+ }
+
resultStream = extractedResult.getResult();
break;
}
@@ -1356,7 +1369,13 @@ public class TestExecutor {
+ ", filectxs.size: " + numResultFiles);
}
} else {
- writeOutputToFile(actualResultFile, resultStream);
+ if (resultVar != null) {
+ String result = IOUtils.toString(resultStream, responseCharset);
+ variableCtx.put(resultVar, result);
+ writeOutputToFile(actualResultFile, new ByteArrayInputStream(result.getBytes(responseCharset)));
+ } else {
+ writeOutputToFile(actualResultFile, resultStream);
+ }
if (expectedResultFile == null) {
if (reqType.equals("store")) {
return extractedResult;
@@ -1584,6 +1603,16 @@ public class TestExecutor {
return handleVariableMatcher.find() ? handleVariableMatcher.group(1) : null;
}
+ protected static String getResultVariable(String statement) {
+ final Matcher resultVariableMatcher = RESULT_VARIABLE_PATTERN.matcher(statement);
+ return resultVariableMatcher.find() ? resultVariableMatcher.group(1) : null;
+ }
+
+ protected static String getOutputFormatVariable(String statement) {
+ final Matcher outputFormatVariableMatcher = OUTPUTFORMAT_VARIABLE_PATTERN.matcher(statement);
+ return outputFormatVariableMatcher.find() ? outputFormatVariableMatcher.group(1) : null;
+ }
+
protected static String replaceVarRef(String statement, Map<String, Object> variableCtx) {
String tmpStmt = statement;
Matcher variableReferenceMatcher = VARIABLE_REF_PATTERN.matcher(tmpStmt);
@@ -1597,6 +1626,21 @@ public class TestExecutor {
return tmpStmt;
}
+ protected static String replaceVarRefRelaxed(String statement, Map<String, Object> variableCtx) {
+ String tmpStmt = statement;
+ Matcher variableReferenceMatcher = VARIABLE_REF_PATTERN.matcher(tmpStmt);
+ while (variableReferenceMatcher.find()) {
+ String var = variableReferenceMatcher.group(1);
+ Object value = variableCtx.get(var);
+ if (value == null) {
+ continue;
+ }
+ tmpStmt = tmpStmt.replace("$" + var, String.valueOf(value));
+ variableReferenceMatcher = VARIABLE_REF_PATTERN.matcher(tmpStmt);
+ }
+ return tmpStmt;
+ }
+
protected static Optional<String> extractMaxResultReads(String statement) {
final Matcher m = MAX_RESULT_READS_PATTERN.matcher(statement);
while (m.find()) {
[asterixdb] 02/05: [NO ISSUE][TEST] Default test charsets to UTF-8
only
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 1c3aa58d3b034f40587dc6edf37d57fc2b3e8afb
Author: Michael Blow <mb...@apache.org>
AuthorDate: Tue Oct 15 11:40:48 2019 -0400
[NO ISSUE][TEST] Default test charsets to UTF-8 only
Change-Id: Ia7278fa522d484a4d686cb6436d7e2f50bd7a170
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/3765
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../org/apache/asterix/test/common/TestExecutor.java | 17 ++++++++---------
.../asterix/test/server/SqlppExecutionNCServiceIT.java | 12 ++++++++++++
2 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index af82754..15711d0 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -64,7 +64,6 @@ import java.util.concurrent.TimeoutException;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.asterix.api.http.server.QueryServiceServlet;
@@ -173,7 +172,7 @@ public class TestExecutor {
private static List<InetSocketAddress> ncEndPointsList = new ArrayList<>();
private static Map<String, InetSocketAddress> replicationAddress;
- private final List<Charset> allCharsets;
+ private List<Charset> allCharsets;
private final Queue<Charset> charsetsRemaining = new ArrayDeque<>();
/*
@@ -199,10 +198,7 @@ public class TestExecutor {
public TestExecutor(List<InetSocketAddress> endpoints) {
this.endpoints = endpoints;
- this.allCharsets = Stream
- .of("UTF-8", "UTF-16", "UTF-16BE", "UTF-16LE", "UTF-32", "UTF-32BE", "UTF-32LE", "x-UTF-32BE-BOM",
- "x-UTF-32LE-BOM", "x-UTF-16LE-BOM")
- .filter(Charset::isSupported).map(Charset::forName).collect(Collectors.toList());
+ this.allCharsets = Collections.singletonList(UTF_8);
}
public void setLibrarian(IExternalUDFLibrarian librarian) {
@@ -669,9 +665,12 @@ public class TestExecutor {
responseCharset, responseCodeValidator, cancellable);
}
- public synchronized void setAvailableCharsets(Charset... charsets) {
- allCharsets.clear();
- allCharsets.addAll(Arrays.asList(charsets));
+ public void setAvailableCharsets(Charset... charsets) {
+ setAvailableCharsets(Arrays.asList(charsets));
+ }
+
+ public synchronized void setAvailableCharsets(List<Charset> charsets) {
+ allCharsets = charsets;
charsetsRemaining.clear();
}
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
index 88ada80..47e6f80 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SqlppExecutionNCServiceIT.java
@@ -15,10 +15,14 @@
package org.apache.asterix.test.server;
import java.io.File;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.BeforeClass;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@@ -38,6 +42,14 @@ public class SqlppExecutionNCServiceIT extends AbstractExecutionIT {
return testArgs;
}
+ @BeforeClass
+ public static void setup() {
+ testExecutor.setAvailableCharsets(Stream
+ .of("UTF-8", "UTF-16", "UTF-16BE", "UTF-16LE", "UTF-32", "UTF-32BE", "UTF-32LE", "x-UTF-32BE-BOM",
+ "x-UTF-32LE-BOM", "x-UTF-16LE-BOM")
+ .filter(Charset::isSupported).map(Charset::forName).collect(Collectors.toList()));
+ }
+
protected static Collection<Object[]> buildTestsInXml(String xmlfile) throws Exception {
Collection<Object[]> testArgs = new ArrayList<Object[]>();
TestCaseContext.Builder b = new TestCaseContext.Builder();