You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2019/02/12 06:58:57 UTC
[asterixdb] 03/04: [NO ISSUE][HTTP] Character encoding fixes
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 6312edf35ee4af4e707b880c3e60ad68f246b5d1
Author: Michael Blow <mb...@apache.org>
AuthorDate: Mon Feb 11 19:15:57 2019 -0500
[NO ISSUE][HTTP] Character encoding fixes
- honor Accept-Charset on response
- honor request encoding on requests
Change-Id: I3c066e2ce190c0f271fa1c421ccff657bedb5a44
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3183
Reviewed-by: Till Westmann <ti...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
---
.../apache/asterix/api/http/server/ApiServlet.java | 20 ++-
.../asterix/api/http/server/ClusterApiServlet.java | 2 +-
.../server/ClusterControllerDetailsApiServlet.java | 2 +-
.../api/http/server/ConnectorApiServlet.java | 2 +-
.../api/http/server/DiagnosticsApiServlet.java | 2 +-
.../server/NodeControllerDetailsApiServlet.java | 2 +-
.../api/http/server/QueryResultApiServlet.java | 13 +-
.../api/http/server/QueryServiceServlet.java | 2 +-
.../api/http/server/QueryStatusApiServlet.java | 10 +-
.../api/http/server/RebalanceApiServlet.java | 2 +-
.../asterix/api/http/server/RestApiServlet.java | 2 +-
.../api/http/server/ShutdownApiServlet.java | 2 +-
.../asterix/api/http/server/StorageApiServlet.java | 2 +-
.../asterix/api/http/server/VersionApiServlet.java | 2 +-
.../api/http/server/QueryWebInterfaceServlet.java | 2 +-
.../cc/web/util/JSONOutputRequestHandler.java | 6 +-
hyracks-fullstack/hyracks/hyracks-http/pom.xml | 16 +++
.../hyracks/http/server/ChunkedResponse.java | 26 +++-
.../apache/hyracks/http/server/FullResponse.java | 17 ++-
.../apache/hyracks/http/server/utils/HttpUtil.java | 96 +++++++++++++-
.../hyracks/test/http/HttpAcceptCharsetTest.java | 90 +++++++++++++
.../{http/test => test/http}/HttpRequestTask.java | 2 +-
.../hyracks/test/http/HttpServerEncodingTest.java | 145 +++++++++++++++++++++
.../{http/test => test/http}/HttpServerTest.java | 13 +-
.../hyracks/{ => test}/http/HttpTestUtil.java | 2 +-
.../{ => test}/http/servlet/ChattyServlet.java | 9 +-
.../http/servlet/CompliantEchoServlet.java} | 49 +------
.../{ => test}/http/servlet/SleepyServlet.java | 4 +-
.../hyracks/util/string/UTF8StringSample.java | 30 +++--
.../hyracks/util/string/UTF8StringUtilTest.java | 2 -
30 files changed, 441 insertions(+), 133 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ApiServlet.java
index 868e4b8..42b5125 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ApiServlet.java
@@ -59,7 +59,6 @@ import org.apache.hyracks.http.server.AbstractServlet;
import org.apache.hyracks.http.server.StaticResourceServlet;
import org.apache.hyracks.http.server.utils.HttpUtil;
import org.apache.hyracks.http.server.utils.HttpUtil.ContentType;
-import org.apache.hyracks.http.server.utils.HttpUtil.Encoding;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -94,6 +93,13 @@ public class ApiServlet extends AbstractServlet {
? aqlCompilationProvider : sqlppCompilationProvider;
IParserFactory parserFactory = compilationProvider.getParserFactory();
+ try {
+ HttpUtil.setContentType(response, ContentType.TEXT_HTML, request);
+ } catch (IOException e) {
+ LOGGER.log(Level.WARN, "Failure setting content type", e);
+ response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
+ return;
+ }
// Output format.
PrintWriter out = response.writer();
OutputFormat format;
@@ -124,14 +130,7 @@ public class ApiServlet extends AbstractServlet {
String printOptimizedLogicalPlanParam = request.getParameter("print-optimized-logical-plan");
String printJob = request.getParameter("print-job");
String executeQuery = request.getParameter("execute-query");
- try {
- response.setStatus(HttpResponseStatus.OK);
- HttpUtil.setContentType(response, ContentType.TEXT_HTML, Encoding.UTF8);
- } catch (IOException e) {
- LOGGER.log(Level.WARN, "Failure setting content type", e);
- response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
- return;
- }
+ response.setStatus(HttpResponseStatus.OK);
try {
IHyracksClientConnection hcc = (IHyracksClientConnection) ctx.get(HYRACKS_CONNECTION_ATTR);
IResultSet resultSet = ServletUtil.getResultSet(hcc, appCtx, ctx);
@@ -173,7 +172,7 @@ public class ApiServlet extends AbstractServlet {
response.setStatus(HttpResponseStatus.OK);
if ("/".equals(requestURI)) {
try {
- HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_HTML, HttpUtil.Encoding.UTF8);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_HTML, request);
} catch (IOException e) {
LOGGER.log(Level.WARN, "Failure setting content type", e);
response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
@@ -206,7 +205,6 @@ public class ApiServlet extends AbstractServlet {
} catch (IOException e) {
LOGGER.log(Level.WARN, "Failure handling request", e);
response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
- return;
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
index a3ad089..ce30637 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
@@ -66,7 +66,7 @@ public class ClusterApiServlet extends AbstractServlet {
@Override
protected void get(IServletRequest request, IServletResponse response) throws IOException {
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
PrintWriter responseWriter = response.writer();
try {
ObjectNode json;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterControllerDetailsApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterControllerDetailsApiServlet.java
index 82ade81..6a91fa3 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterControllerDetailsApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterControllerDetailsApiServlet.java
@@ -48,6 +48,7 @@ public class ClusterControllerDetailsApiServlet extends ClusterApiServlet {
@Override
protected void get(IServletRequest request, IServletResponse response) throws IOException {
+ HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
PrintWriter responseWriter = response.writer();
IHyracksClientConnection hcc = (IHyracksClientConnection) ctx.get(HYRACKS_CONNECTION_ATTR);
try {
@@ -58,7 +59,6 @@ public class ClusterControllerDetailsApiServlet extends ClusterApiServlet {
} else {
json = processNode(request, hcc);
}
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
responseWriter.write(OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(json));
} catch (IllegalArgumentException e) { // NOSONAR - exception not logged or rethrown
response.setStatus(HttpResponseStatus.NOT_FOUND);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java
index 5fe88f0..e422c24 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java
@@ -69,7 +69,7 @@ public class ConnectorApiServlet extends AbstractServlet {
protected void get(IServletRequest request, IServletResponse response) {
response.setStatus(HttpResponseStatus.OK);
try {
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
} catch (IOException e) {
LOGGER.log(Level.WARN, "Failure setting content type", e);
response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/DiagnosticsApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/DiagnosticsApiServlet.java
index c4bd054..e294510 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/DiagnosticsApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/DiagnosticsApiServlet.java
@@ -63,7 +63,7 @@ public class DiagnosticsApiServlet extends NodeControllerDetailsApiServlet {
@Override
protected void get(IServletRequest request, IServletResponse response) throws IOException {
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
PrintWriter responseWriter = response.writer();
response.setStatus(HttpResponseStatus.OK);
try {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NodeControllerDetailsApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NodeControllerDetailsApiServlet.java
index 09a07a1..a5ae552 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NodeControllerDetailsApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NodeControllerDetailsApiServlet.java
@@ -52,6 +52,7 @@ public class NodeControllerDetailsApiServlet extends ClusterApiServlet {
@Override
protected void get(IServletRequest request, IServletResponse response) throws IOException {
+ HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
PrintWriter responseWriter = response.writer();
IHyracksClientConnection hcc = (IHyracksClientConnection) ctx.get(HYRACKS_CONNECTION_ATTR);
try {
@@ -63,7 +64,6 @@ public class NodeControllerDetailsApiServlet extends ClusterApiServlet {
} else {
json = processNode(request, hcc);
}
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
responseWriter.write(OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(json));
} catch (IllegalStateException e) { // NOSONAR - exception not logged or rethrown
response.setStatus(HttpResponseStatus.SERVICE_UNAVAILABLE);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
index 6781f22..8fdcc41 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.api.http.server;
-import java.io.PrintWriter;
import java.util.concurrent.ConcurrentMap;
import org.apache.asterix.app.result.ResultHandle;
@@ -26,10 +25,10 @@ import org.apache.asterix.app.result.ResultReader;
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.translator.IStatementExecutor.Stats;
import org.apache.asterix.translator.SessionOutput;
-import org.apache.hyracks.api.result.ResultJobRecord;
-import org.apache.hyracks.api.result.IResultSet;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.result.IResultSet;
+import org.apache.hyracks.api.result.ResultJobRecord;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.utils.HttpUtil;
@@ -48,9 +47,7 @@ public class QueryResultApiServlet extends AbstractQueryApiServlet {
@Override
protected void get(IServletRequest request, IServletResponse response) throws Exception {
- // TODO this seems wrong ...
- HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_HTML, HttpUtil.Encoding.UTF8);
- PrintWriter out = response.writer();
+ HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_HTML, request);
final String strHandle = localPath(request);
final ResultHandle handle = ResultHandle.parse(strHandle);
@@ -104,13 +101,13 @@ public class QueryResultApiServlet extends AbstractQueryApiServlet {
return;
}
response.setStatus(HttpResponseStatus.BAD_REQUEST);
- out.println(e.getMessage());
+ response.writer().println(e.getMessage());
LOGGER.log(Level.WARN, "Error retrieving result for \"" + strHandle + "\"", e);
} catch (Exception e) {
response.setStatus(HttpResponseStatus.BAD_REQUEST);
LOGGER.log(Level.WARN, "Error retrieving result for \"" + strHandle + "\"", e);
}
- if (out.checkError()) {
+ if (response.writer().checkError()) {
LOGGER.warn("Error flushing output writer for \"" + strHandle + "\"");
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 765ba9c..625834f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -512,6 +512,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
QueryServiceRequestParameters param = getRequestParameters(request);
LOGGER.info("handleRequest: {}", param);
long elapsedStart = System.nanoTime();
+ HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
final PrintWriter httpWriter = response.writer();
ResultDelivery delivery = parseResultDelivery(param.getMode());
@@ -522,7 +523,6 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
String handleUrl = getHandleUrl(param.getHost(), param.getPath(), delivery);
SessionOutput sessionOutput = createSessionOutput(param, handleUrl, httpWriter);
SessionConfig sessionConfig = sessionOutput.config();
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
Stats stats = new Stats();
RequestExecutionState execution = new RequestExecutionState();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
index d983bfd..df09aee 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
@@ -21,7 +21,6 @@ package org.apache.asterix.api.http.server;
import static org.apache.asterix.api.http.server.AbstractQueryApiServlet.ResultStatus.FAILED;
import java.io.PrintWriter;
-import java.io.StringWriter;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
@@ -66,10 +65,9 @@ public class QueryStatusApiServlet extends AbstractQueryApiServlet {
ResultStatus resultStatus = resultStatus(resultReaderStatus);
Exception ex = extractException(resultReaderStatus);
- final StringWriter stringWriter = new StringWriter();
- final PrintWriter resultWriter = new PrintWriter(stringWriter);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
+ final PrintWriter resultWriter = response.writer();
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
HttpResponseStatus httpStatus = HttpResponseStatus.OK;
resultWriter.print("{\n");
@@ -84,11 +82,7 @@ public class QueryStatusApiServlet extends AbstractQueryApiServlet {
}
resultWriter.print("}\n");
- resultWriter.flush();
- String result = stringWriter.toString();
-
response.setStatus(httpStatus);
- response.writer().print(result);
if (response.writer().checkError()) {
LOGGER.warn("Error flushing output writer");
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java
index 51f420c..320c7aa 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RebalanceApiServlet.java
@@ -87,7 +87,7 @@ public class RebalanceApiServlet extends AbstractServlet {
protected void delete(IServletRequest request, IServletResponse response) {
try {
// Sets the content type.
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
// Cancels all rebalance requests.
cancelRebalance();
// Sends the response back.
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
index 99c7308..347b2d7 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
@@ -86,7 +86,7 @@ public abstract class RestApiServlet extends AbstractServlet {
* based on the Accept: header and other servlet parameters.
*/
static SessionOutput initResponse(IServletRequest request, IServletResponse response) throws IOException {
- HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_PLAIN, HttpUtil.Encoding.UTF8);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_PLAIN, request);
// CLEAN_JSON output is the default; most generally useful for a
// programmatic HTTP API
OutputFormat format = OutputFormat.CLEAN_JSON;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ShutdownApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ShutdownApiServlet.java
index 8c4f22d..a228f93 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ShutdownApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ShutdownApiServlet.java
@@ -71,7 +71,7 @@ public class ShutdownApiServlet extends AbstractServlet {
}, "Shutdown Servlet Worker");
try {
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
} catch (IOException e) {
LOGGER.log(Level.WARN, "Failure handling request", e);
response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
index 6b632a1..eaaa0f6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
@@ -63,7 +63,7 @@ public class StorageApiServlet extends AbstractServlet {
@Override
protected void get(IServletRequest request, IServletResponse response) throws IOException {
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
PrintWriter responseWriter = response.writer();
try {
JsonNode json;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/VersionApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/VersionApiServlet.java
index 8b615cf..80805ce 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/VersionApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/VersionApiServlet.java
@@ -53,7 +53,7 @@ public class VersionApiServlet extends AbstractServlet {
ObjectNode responseObject = OBJECT_MAPPER.createObjectNode();
buildProperties.forEach(responseObject::put);
try {
- HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_PLAIN, HttpUtil.Encoding.UTF8);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_PLAIN, request);
} catch (IOException e) {
LOGGER.log(Level.WARN, "Failure handling request", e);
response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
diff --git a/asterixdb/asterix-dashboard/src/main/java/org/apache/asterix/api/http/server/QueryWebInterfaceServlet.java b/asterixdb/asterix-dashboard/src/main/java/org/apache/asterix/api/http/server/QueryWebInterfaceServlet.java
index 900e72b..3ba3de9 100644
--- a/asterixdb/asterix-dashboard/src/main/java/org/apache/asterix/api/http/server/QueryWebInterfaceServlet.java
+++ b/asterixdb/asterix-dashboard/src/main/java/org/apache/asterix/api/http/server/QueryWebInterfaceServlet.java
@@ -59,7 +59,7 @@ public class QueryWebInterfaceServlet extends StaticResourceServlet {
@Override
protected void post(IServletRequest request, IServletResponse response) throws IOException {
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
ExternalProperties externalProperties = appCtx.getExternalProperties();
response.setStatus(HttpResponseStatus.OK);
ObjectNode obj = OBJECT_MAPPER.createObjectNode();
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/web/util/JSONOutputRequestHandler.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/web/util/JSONOutputRequestHandler.java
index 479004d..4fd3496 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/web/util/JSONOutputRequestHandler.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/web/util/JSONOutputRequestHandler.java
@@ -56,7 +56,7 @@ public class JSONOutputRequestHandler extends AbstractServlet {
ObjectNode result = invoke(response, host, servletPath, parts);
if (result != null) {
- deliver(response, result);
+ deliver(request, response, result);
}
}
@@ -71,9 +71,9 @@ public class JSONOutputRequestHandler extends AbstractServlet {
return null;
}
- protected void deliver(IServletResponse response, ObjectNode result) {
+ protected void deliver(IServletRequest request, IServletResponse response, ObjectNode result) {
try {
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
ObjectMapper om = new ObjectMapper();
om.writer().writeValue(response.writer(), result);
response.setStatus(HttpResponseStatus.OK);
diff --git a/hyracks-fullstack/hyracks/hyracks-http/pom.xml b/hyracks-fullstack/hyracks/hyracks-http/pom.xml
index 6ca7a42..a67fa15 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-http/pom.xml
@@ -79,5 +79,21 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java
index a67b40e..e72bee9 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java
@@ -18,9 +18,13 @@
*/
package org.apache.hyracks.http.server;
+import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStream;
+import java.io.OutputStreamWriter;
import java.io.PrintWriter;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.logging.log4j.Level;
@@ -29,7 +33,6 @@ import org.apache.logging.log4j.Logger;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.http.DefaultFullHttpResponse;
import io.netty.handler.codec.http.DefaultHttpResponse;
@@ -64,7 +67,7 @@ public class ChunkedResponse implements IServletResponse {
private static final Logger LOGGER = LogManager.getLogger();
private final ChannelHandlerContext ctx;
private final ChunkedNettyOutputStream outputStream;
- private final PrintWriter writer;
+ private PrintWriter writer;
private HttpResponse response;
private boolean headerSent;
private ByteBuf error;
@@ -75,7 +78,6 @@ public class ChunkedResponse implements IServletResponse {
public ChunkedResponse(ChannelHandlerContext ctx, FullHttpRequest request, int chunkSize) {
this.ctx = ctx;
outputStream = new ChunkedNettyOutputStream(ctx, chunkSize, this);
- writer = new PrintWriter(outputStream);
response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR);
response.headers().set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED);
keepAlive = HttpUtil.isKeepAlive(request);
@@ -88,7 +90,11 @@ public class ChunkedResponse implements IServletResponse {
if (headerSent) {
throw new IOException("Can't add more headers since the initial response was sent");
}
- response.headers().set(name, value);
+ String nameString = String.valueOf(name);
+ if (writer != null && nameString.equals(HttpHeaderNames.CONTENT_TYPE.toString())) {
+ throw new IOException("Can't set " + HttpHeaderNames.CONTENT_TYPE + " after writer has been accessed");
+ }
+ response.headers().set(nameString, value);
return this;
}
@@ -98,13 +104,21 @@ public class ChunkedResponse implements IServletResponse {
}
@Override
- public PrintWriter writer() {
+ public synchronized PrintWriter writer() {
+ if (writer == null) {
+ Charset charset = io.netty.handler.codec.http.HttpUtil.getCharset(response, StandardCharsets.UTF_8);
+ writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(outputStream, charset)));
+ }
return writer;
}
@Override
public void close() throws IOException {
- writer.close();
+ if (writer != null) {
+ writer.close();
+ } else {
+ outputStream.close();
+ }
if (error == null && response.status() == HttpResponseStatus.OK) {
if (!done) {
future = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java
index 1d28472..85a0a43 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java
@@ -18,10 +18,14 @@
*/
package org.apache.hyracks.http.server;
+import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.io.OutputStreamWriter;
import java.io.PrintWriter;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import org.apache.hyracks.http.api.IServletResponse;
@@ -41,15 +45,14 @@ import io.netty.handler.codec.http.HttpVersion;
public class FullResponse implements IServletResponse {
private final ChannelHandlerContext ctx;
private final ByteArrayOutputStream baos;
- private final PrintWriter writer;
private final FullHttpResponse response;
private final boolean keepAlive;
+ private PrintWriter writer;
private ChannelFuture future;
public FullResponse(ChannelHandlerContext ctx, FullHttpRequest request) {
this.ctx = ctx;
baos = new ByteArrayOutputStream();
- writer = new PrintWriter(baos);
response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR);
keepAlive = HttpUtil.isKeepAlive(request);
if (keepAlive) {
@@ -59,7 +62,9 @@ public class FullResponse implements IServletResponse {
@Override
public void close() throws IOException {
- writer.close();
+ if (writer != null) {
+ writer.close();
+ }
FullHttpResponse fullResponse = response.replace(Unpooled.copiedBuffer(baos.toByteArray()));
if (keepAlive) {
if (response.status() == HttpResponseStatus.OK || response.status() == HttpResponseStatus.UNAUTHORIZED) {
@@ -81,7 +86,11 @@ public class FullResponse implements IServletResponse {
}
@Override
- public PrintWriter writer() {
+ public synchronized PrintWriter writer() {
+ if (writer == null) {
+ Charset charset = io.netty.handler.codec.http.HttpUtil.getCharset(response, StandardCharsets.UTF_8);
+ writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(baos, charset)));
+ }
return writer;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
index 8d6dfbc..b34b9dc 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
@@ -19,23 +19,33 @@
package org.apache.hyracks.http.server.utils;
import java.io.IOException;
+import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.OptionalDouble;
import java.util.regex.Pattern;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.BaseRequest;
import org.apache.hyracks.http.server.FormUrlEncodedRequest;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.HttpRequest;
public class HttpUtil {
+ private static final Logger LOGGER = LogManager.getLogger();
private static final Pattern PARENT_DIR = Pattern.compile("/[^./]+/\\.\\./");
+ private static final String DEFAULT_RESPONSE_CHARSET = StandardCharsets.UTF_8.name();
private HttpUtil() {
}
@@ -91,7 +101,14 @@ public class HttpUtil {
}
public static String getRequestBody(IServletRequest request) {
- return request.getHttpRequest().content().toString(StandardCharsets.UTF_8);
+ FullHttpRequest httpRequest = request.getHttpRequest();
+ Charset charset = io.netty.handler.codec.http.HttpUtil.getCharset(httpRequest, StandardCharsets.UTF_8);
+ return httpRequest.content().toString(charset);
+ }
+
+ public static void setContentType(IServletResponse response, String type, IServletRequest fromRequest)
+ throws IOException {
+ response.setHeader(HttpHeaderNames.CONTENT_TYPE, type + "; charset=" + getPreferredCharset(fromRequest));
}
public static void setContentType(IServletResponse response, String type, String charset) throws IOException {
@@ -104,9 +121,7 @@ public class HttpUtil {
public static Map<String, String> getRequestHeaders(IServletRequest request) {
Map<String, String> headers = new HashMap<>();
- request.getHttpRequest().headers().forEach(entry -> {
- headers.put(entry.getKey(), entry.getValue());
- });
+ request.getHttpRequest().headers().forEach(entry -> headers.put(entry.getKey(), entry.getValue()));
return headers;
}
@@ -154,4 +169,77 @@ public class HttpUtil {
return clusterURL;
}
+ public static String getPreferredCharset(IServletRequest request) {
+ return getPreferredCharset(request, DEFAULT_RESPONSE_CHARSET);
+ }
+
+ public static String getPreferredCharset(IServletRequest request, String defaultCharset) {
+ String acceptCharset = request.getHeader(HttpHeaderNames.ACCEPT_CHARSET);
+ if (acceptCharset == null) {
+ return defaultCharset;
+ }
+ // If no "q" parameter is present, the default weight is 1 [https://tools.ietf.org/html/rfc7231#section-5.3.1]
+ Optional<String> preferredCharset = Stream.of(StringUtils.split(acceptCharset, ","))
+ .map(WeightedHeaderValue::new).sorted().map(WeightedHeaderValue::getValue)
+ .map(a -> "*".equals(a) ? defaultCharset : a).filter(value -> {
+ if (!Charset.isSupported(value)) {
+ LOGGER.info("disregarding unsupported charset '{}'", value);
+ return false;
+ }
+ return true;
+ }).findFirst();
+ return preferredCharset.orElse(defaultCharset);
+ }
+
+ private static class WeightedHeaderValue implements Comparable<WeightedHeaderValue> {
+
+ final String value;
+ final double weight;
+
+ WeightedHeaderValue(String value) {
+ // Accept-Charset = 1#( ( charset / "*" ) [ weight ] )
+ // weight = OWS ";" OWS "q=" qvalue
+ String[] splits = StringUtils.split(value, ";");
+ this.value = splits[0].trim();
+ if (splits.length == 1) {
+ weight = 1.0d;
+ } else {
+ OptionalDouble specifiedWeight = Stream.of(splits).skip(1).map(String::trim).map(String::toLowerCase)
+ .filter(a -> a.startsWith("q="))
+ .mapToDouble(segment -> Double.parseDouble(StringUtils.splitByWholeSeparator(segment, "q=")[0]))
+ .findFirst();
+ this.weight = specifiedWeight.orElse(1.0d);
+ }
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public double getWeight() {
+ return weight;
+ }
+
+ @Override
+ public int compareTo(WeightedHeaderValue o) {
+ return Double.compare(o.weight, weight);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ WeightedHeaderValue that = (WeightedHeaderValue) o;
+ return Double.compare(that.weight, weight) == 0 && Objects.equals(value, that.value);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(value, weight);
+ }
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpAcceptCharsetTest.java b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpAcceptCharsetTest.java
new file mode 100644
index 0000000..a166e52
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpAcceptCharsetTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.test.http;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.hyracks.http.server.utils.HttpUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import io.netty.handler.codec.http.HttpHeaderNames;
+
+public class HttpAcceptCharsetTest {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ @Test
+ public void testBogusAcceptCharset() {
+ IServletRequest request = withCharset("lol");
+ assertCharsetsEqual("default on bogus", StandardCharsets.UTF_8, request);
+ }
+
+ @Test
+ public void testWeightedStarDefault() {
+ IServletRequest request = withCharset("*;q=0.5,bogus;q=1.0");
+ assertCharsetsEqual("defer on unsupported", StandardCharsets.UTF_8, request);
+ }
+
+ @Test
+ public void testWeightedStar() {
+ IServletRequest request = withCharset("*;q=0.5,utf-32;q=1.0");
+ assertCharsetsEqual("defer on bogus", Charset.forName("utf-32"), request);
+ }
+
+ @Test
+ public void testUnweightedIs1_0() {
+ IServletRequest request = withCharset("utf-8;q=0.5,utf-16");
+ assertCharsetsEqual("defer on bogus", Charset.forName("utf-16"), request);
+ }
+
+ @Test
+ public void testAmbiguous() {
+ IServletRequest request = withCharset("utf-8;q=.75,utf-16;q=.75,utf-32;q=.5");
+ String preferredCharset = HttpUtil.getPreferredCharset(request);
+ Assert.assertTrue("ambiguous by weight (got: " + preferredCharset + ")",
+ preferredCharset.toLowerCase().matches("utf-(8|16)"));
+ }
+
+ private IServletRequest withCharset(String headerValue) {
+ IServletRequest request = Mockito.mock(IServletRequest.class);
+ Mockito.when(request.getHeader(HttpHeaderNames.ACCEPT_CHARSET)).thenReturn(headerValue);
+ LOGGER.info("using {} of '{}'", HttpHeaderNames.ACCEPT_CHARSET, headerValue);
+ return request;
+ }
+
+ private void assertCharsetsEqual(String message, Object charset, Object charset2) {
+ Assert.assertEquals(message, normalize(charset), normalize(charset2));
+ }
+
+ private String normalize(Object charsetObject) {
+ if (charsetObject instanceof Charset) {
+ return ((Charset) charsetObject).name().toLowerCase();
+ } else if (charsetObject instanceof String) {
+ return ((String) charsetObject).toLowerCase();
+ } else if (charsetObject instanceof IServletRequest) {
+ return HttpUtil.getPreferredCharset((IServletRequest) charsetObject).toLowerCase();
+ }
+ throw new IllegalArgumentException("unknown type: " + charsetObject.getClass());
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/test/HttpRequestTask.java b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpRequestTask.java
similarity index 99%
rename from hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/test/HttpRequestTask.java
rename to hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpRequestTask.java
index 78226ae..516273b 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/test/HttpRequestTask.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpRequestTask.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.http.test;
+package org.apache.hyracks.test.http;
import java.io.BufferedReader;
import java.io.InputStream;
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerEncodingTest.java b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerEncodingTest.java
new file mode 100644
index 0000000..c71c5e4
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerEncodingTest.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.test.http;
+
+import static org.apache.hyracks.util.string.UTF8StringSample.STRING_NEEDS_2_JAVA_CHARS_1;
+import static org.apache.hyracks.util.string.UTF8StringSample.STRING_NEEDS_2_JAVA_CHARS_2;
+
+import java.net.URI;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.hyracks.http.server.HttpServer;
+import org.apache.hyracks.http.server.HttpServerConfig;
+import org.apache.hyracks.http.server.HttpServerConfigBuilder;
+import org.apache.hyracks.http.server.WebManager;
+import org.apache.hyracks.test.http.servlet.CompliantEchoServlet;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class HttpServerEncodingTest {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private static final int PORT = 9898;
+ private static final String HOST = "localhost";
+ private static final String PROTOCOL = "http";
+ private static final String PATH = "/";
+ private static WebManager webMgr;
+ private static CloseableHttpClient client;
+ private static URI uri;
+
+ @Parameters(name = "HttpServerEncodingTest {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ List<Object[]> tests = new ArrayList<>();
+ Stream.of("encoding is hard", "中文字符", "لا يوجد ترجمة لكُ", STRING_NEEDS_2_JAVA_CHARS_1,
+ STRING_NEEDS_2_JAVA_CHARS_2).forEach(input -> {
+ Set<Charset> legalCharsets = getLegalCharsetsFor(input);
+ legalCharsets.forEach(charsetIn -> legalCharsets.forEach(charsetOut -> tests
+ .add(new Object[] { input + ":" + charsetIn.displayName() + "->" + charsetOut.displayName(),
+ input, charsetIn, charsetOut })));
+ });
+ return tests;
+ }
+
+ private static Set<Charset> getLegalCharsetsFor(String input) {
+ return Charset.availableCharsets().values().stream().filter(Charset::canEncode)
+ .filter(test -> canEncodeDecode(input, test)).collect(Collectors.toSet());
+ }
+
+ private static boolean canEncodeDecode(String input, Charset charset) {
+ if (input.equals(new String(input.getBytes(charset), charset))) {
+ return true;
+ }
+ LOGGER.info("cannot encode / decode {} with {}", input, charset.displayName());
+ return false;
+ }
+
+ @Parameter(0)
+ public String testName;
+
+ @Parameter(1)
+ public String input;
+
+ @Parameter(2)
+ public Charset inputCharset;
+
+ @Parameter(3)
+ public Charset outputCharset;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ int numExecutors = 16;
+ int serverQueueSize = 16;
+ webMgr = new WebManager();
+ final HttpServerConfig config = HttpServerConfigBuilder.custom().setThreadCount(numExecutors)
+ .setRequestQueueSize(serverQueueSize).build();
+ HttpServer server = new HttpServer(webMgr.getBosses(), webMgr.getWorkers(), PORT, config);
+ CompliantEchoServlet servlet = new CompliantEchoServlet(server.ctx(), PATH);
+ server.addServlet(servlet);
+ webMgr.add(server);
+ webMgr.start();
+ client = HttpClients.custom().build();
+ uri = new URI(PROTOCOL, null, HOST, PORT, PATH, null, null);
+
+ }
+
+ @AfterClass
+ public static void teardown() throws Exception {
+ client.close();
+ webMgr.stop();
+ }
+
+ @Test
+ public void testAcceptCharset() throws Exception {
+ RequestBuilder builder = RequestBuilder.post(uri);
+ builder.setEntity(new StringEntity(input, inputCharset));
+ builder.setCharset(inputCharset);
+ builder.setHeader(HttpHeaders.ACCEPT_CHARSET, inputCharset.name());
+ try (CloseableHttpResponse response = client.execute(builder.build())) {
+ HttpEntity entity = response.getEntity();
+ Assert.assertEquals(String.valueOf(inputCharset), ContentType.getOrDefault(entity).getCharset(),
+ inputCharset);
+ Assert.assertEquals(String.valueOf(inputCharset), input, EntityUtils.toString(entity));
+ }
+ }
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/test/HttpServerTest.java b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerTest.java
similarity index 97%
rename from hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/test/HttpServerTest.java
rename to hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerTest.java
index b5683ae..ea52b0b 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/test/HttpServerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerTest.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.http.test;
+package org.apache.hyracks.test.http;
import java.io.BufferedReader;
import java.io.InputStreamReader;
@@ -32,14 +32,13 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.hyracks.http.HttpTestUtil;
import org.apache.hyracks.http.server.HttpServer;
import org.apache.hyracks.http.server.HttpServerConfig;
import org.apache.hyracks.http.server.HttpServerConfigBuilder;
import org.apache.hyracks.http.server.InterruptOnCloseHandler;
import org.apache.hyracks.http.server.WebManager;
-import org.apache.hyracks.http.servlet.ChattyServlet;
-import org.apache.hyracks.http.servlet.SleepyServlet;
+import org.apache.hyracks.test.http.servlet.ChattyServlet;
+import org.apache.hyracks.test.http.servlet.SleepyServlet;
import org.apache.hyracks.util.StorageUtil;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -370,12 +369,6 @@ public class HttpServerTest {
Assert.assertNotNull(failure);
}
- public static void setPrivateField(Object obj, String filedName, Object value) throws Exception {
- Field f = obj.getClass().getDeclaredField(filedName);
- f.setAccessible(true);
- f.set(obj, value);
- }
-
private void request(int count) throws URISyntaxException {
request(count, 0);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/HttpTestUtil.java b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpTestUtil.java
similarity index 99%
rename from hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/HttpTestUtil.java
rename to hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpTestUtil.java
index 16efc07..af5e3e3 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/HttpTestUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpTestUtil.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.http;
+package org.apache.hyracks.test.http;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryPoolMXBean;
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/servlet/ChattyServlet.java b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/servlet/ChattyServlet.java
similarity index 91%
rename from hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/servlet/ChattyServlet.java
rename to hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/servlet/ChattyServlet.java
index 54bb756..c61daa9 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/servlet/ChattyServlet.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/servlet/ChattyServlet.java
@@ -16,11 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.http.servlet;
+package org.apache.hyracks.test.http.servlet;
+import java.nio.charset.StandardCharsets;
import java.util.concurrent.ConcurrentMap;
-import org.apache.hyracks.http.HttpTestUtil;
+import org.apache.hyracks.test.http.HttpTestUtil;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.AbstractServlet;
@@ -43,7 +44,7 @@ public class ChattyServlet extends AbstractServlet {
responseBuilder.append(line);
}
String responseString = responseBuilder.toString();
- bytes = responseString.getBytes();
+ bytes = responseString.getBytes(StandardCharsets.UTF_8);
}
@Override
@@ -54,7 +55,7 @@ public class ChattyServlet extends AbstractServlet {
@Override
protected void get(IServletRequest request, IServletResponse response) throws Exception {
response.setStatus(HttpResponseStatus.OK);
- HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_HTML, HttpUtil.Encoding.UTF8);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_HTML, request);
LOGGER.log(Level.WARN, "I am about to flood you... and a single buffer is " + bytes.length + " bytes");
for (int i = 0; i < 100; i++) {
response.outputStream().write(bytes);
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/servlet/SleepyServlet.java b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/servlet/CompliantEchoServlet.java
similarity index 50%
copy from hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/servlet/SleepyServlet.java
copy to hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/servlet/CompliantEchoServlet.java
index 2a5a0a9..634c215 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/servlet/SleepyServlet.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/servlet/CompliantEchoServlet.java
@@ -16,9 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.http.servlet;
+package org.apache.hyracks.test.http.servlet;
-import java.nio.charset.StandardCharsets;
import java.util.concurrent.ConcurrentMap;
import org.apache.hyracks.http.api.IServletRequest;
@@ -26,54 +25,16 @@ import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.AbstractServlet;
import org.apache.hyracks.http.server.utils.HttpUtil;
-import io.netty.handler.codec.http.HttpResponseStatus;
+public class CompliantEchoServlet extends AbstractServlet {
-public class SleepyServlet extends AbstractServlet {
-
- private volatile boolean sleep = true;
- private int numSlept = 0;
-
- public SleepyServlet(ConcurrentMap<String, Object> ctx, String[] paths) {
+ public CompliantEchoServlet(ConcurrentMap<String, Object> ctx, String... paths) {
super(ctx, paths);
}
@Override
protected void post(IServletRequest request, IServletResponse response) throws Exception {
- get(request, response);
- }
-
- @Override
- protected void get(IServletRequest request, IServletResponse response) throws Exception {
- response.setStatus(HttpResponseStatus.OK);
- if (sleep) {
- synchronized (this) {
- if (sleep) {
- incrementSleptCount();
- while (sleep) {
- this.wait();
- }
- }
- }
- }
- HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_HTML, HttpUtil.Encoding.UTF8);
- response.outputStream().write("I am playing hard to get".getBytes(StandardCharsets.UTF_8));
- }
-
- private void incrementSleptCount() {
- numSlept++;
- notifyAll();
- }
-
- public int getNumSlept() {
- return numSlept;
+ HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_HTML, request);
+ response.writer().write(HttpUtil.getRequestBody(request));
}
- public synchronized void wakeUp() {
- sleep = false;
- notifyAll();
- }
-
- public void sleep() {
- sleep = true;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/servlet/SleepyServlet.java b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/servlet/SleepyServlet.java
similarity index 96%
rename from hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/servlet/SleepyServlet.java
rename to hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/servlet/SleepyServlet.java
index 2a5a0a9..89cac8f 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/http/servlet/SleepyServlet.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/servlet/SleepyServlet.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.http.servlet;
+package org.apache.hyracks.test.http.servlet;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.ConcurrentMap;
@@ -55,7 +55,7 @@ public class SleepyServlet extends AbstractServlet {
}
}
}
- HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_HTML, HttpUtil.Encoding.UTF8);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_HTML, request);
response.outputStream().write("I am playing hard to get".getBytes(StandardCharsets.UTF_8));
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringSample.java b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringSample.java
index 3e6e984..6984d75 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringSample.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringSample.java
@@ -28,24 +28,26 @@ import java.util.Arrays;
* Util class to provide the sample test string
*/
public class UTF8StringSample {
- public static String EMPTY_STRING = "";
+ public static final String EMPTY_STRING = "";
- public static char ONE_ASCII_CHAR = 'x';
- public static char ONE_UTF8_CHAR = 'à';
+ public static final char ONE_ASCII_CHAR = 'x';
+ public static final char ONE_UTF8_CHAR = 'à';
- public static String STRING_LEN_3 = "xyz";
- public static String STRING_UTF8_3 = "锟斤拷";
- public static String STRING_UTF8_MIX = "\uD841\uDF0E\uD841\uDF31锟X斤Y拷Zà"; // one, two, three, and four bytes
- public static String STRING_UTF8_MIX_LOWERCASE = "\uD841\uDF0E\uD841\uDF31锟x斤y拷zà";
+ public static final String STRING_LEN_3 = "xyz";
+ public static final String STRING_UTF8_3 = "锟斤拷";
+ public static final String STRING_UTF8_MIX = "\uD841\uDF0E\uD841\uDF31锟X斤Y拷Zà"; // one, two, three, and four bytes
+ public static final String STRING_UTF8_MIX_LOWERCASE = "\uD841\uDF0E\uD841\uDF31锟x斤y拷zà";
+ public static final String STRING_NEEDS_2_JAVA_CHARS_1 = "\uD83D\uDE22\uD83D\uDE22\uD83D\uDC89\uD83D\uDC89";
+ public static final String STRING_NEEDS_2_JAVA_CHARS_2 = "😢😢💉💉";
- public static String STRING_LEN_127 = generateStringRepeatBy(ONE_ASCII_CHAR, 127);
- public static String STRING_LEN_128 = generateStringRepeatBy(ONE_ASCII_CHAR, 128);
+ public static final String STRING_LEN_127 = generateStringRepeatBy(ONE_ASCII_CHAR, 127);
+ public static final String STRING_LEN_128 = generateStringRepeatBy(ONE_ASCII_CHAR, 128);
- public static String STRING_LEN_MEDIUM_SUB_1 = generateStringRepeatBy(ONE_ASCII_CHAR, BOUND_TWO_BYTE - 1);
- public static String STRING_LEN_MEDIUM = generateStringRepeatBy(ONE_ASCII_CHAR, BOUND_TWO_BYTE);
+ public static final String STRING_LEN_MEDIUM_SUB_1 = generateStringRepeatBy(ONE_ASCII_CHAR, BOUND_TWO_BYTE - 1);
+ public static final String STRING_LEN_MEDIUM = generateStringRepeatBy(ONE_ASCII_CHAR, BOUND_TWO_BYTE);
- public static String STRING_LEN_LARGE_SUB_1 = generateStringRepeatBy(ONE_ASCII_CHAR, BOUND_THREE_BYTE - 1);
- public static String STRING_LEN_LARGE = generateStringRepeatBy(ONE_ASCII_CHAR, BOUND_THREE_BYTE);
+ public static final String STRING_LEN_LARGE_SUB_1 = generateStringRepeatBy(ONE_ASCII_CHAR, BOUND_THREE_BYTE - 1);
+ public static final String STRING_LEN_LARGE = generateStringRepeatBy(ONE_ASCII_CHAR, BOUND_THREE_BYTE);
public static String generateStringRepeatBy(char c, int times) {
char[] chars = new char[times];
@@ -53,4 +55,6 @@ public class UTF8StringSample {
return new String(chars);
}
+ private UTF8StringSample() {
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringUtilTest.java b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringUtilTest.java
index 5a614f0..c48e1d8 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringUtilTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringUtilTest.java
@@ -38,13 +38,11 @@ import static org.apache.hyracks.util.string.UTF8StringUtil.lowerCaseHash;
import static org.apache.hyracks.util.string.UTF8StringUtil.normalize;
import static org.apache.hyracks.util.string.UTF8StringUtil.rawByteCompareTo;
import static org.apache.hyracks.util.string.UTF8StringUtil.hash;
-import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
-import org.junit.Assert;
import org.junit.Test;
public class UTF8StringUtilTest {