You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2022/04/22 15:21:53 UTC
[nifi] 01/02: NIFI-9902 - Improve handling of charsets when performing string/byte conversions
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.16
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 2b6af1a2cb26752ad3c0a4ae7898352299f043de
Author: Paul Grey <gr...@yahoo.com>
AuthorDate: Mon Apr 11 11:35:37 2022 -0400
NIFI-9902 - Improve handling of charsets when performing string/byte conversions
This closes #5954
Signed-off-by: Mike Thomsen <mt...@apache.org>
---
nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java | 2 +-
.../apache/nifi/jasn1/TestJASN1RecordReaderWithSimpleTypes.java | 3 ++-
.../org/apache/nifi/serialization/record/MockRecordWriter.java | 9 +++++----
.../apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java | 5 +++--
.../apache/nifi/processors/slack/PostSlackTextMessageTest.java | 4 +++-
5 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java
index 7ee30a7c00..36c95652eb 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java
@@ -272,7 +272,7 @@ public class MockFlowFile implements FlowFileRecord {
}
public void assertContentEquals(final String data) {
- assertContentEquals(data, "UTF-8");
+ assertContentEquals(data, StandardCharsets.UTF_8);
}
public void assertContentEquals(final String data, final String charset) {
diff --git a/nifi-nar-bundles/nifi-asn1-bundle/nifi-asn1-services/src/test/java/org/apache/nifi/jasn1/TestJASN1RecordReaderWithSimpleTypes.java b/nifi-nar-bundles/nifi-asn1-bundle/nifi-asn1-services/src/test/java/org/apache/nifi/jasn1/TestJASN1RecordReaderWithSimpleTypes.java
index c6cb13a638..67cf5c6419 100644
--- a/nifi-nar-bundles/nifi-asn1-bundle/nifi-asn1-services/src/test/java/org/apache/nifi/jasn1/TestJASN1RecordReaderWithSimpleTypes.java
+++ b/nifi-nar-bundles/nifi-asn1-bundle/nifi-asn1-services/src/test/java/org/apache/nifi/jasn1/TestJASN1RecordReaderWithSimpleTypes.java
@@ -46,6 +46,7 @@ import org.apache.nifi.serialization.record.RecordSchema;
import org.junit.jupiter.api.Test;
import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
@@ -156,7 +157,7 @@ public class TestJASN1RecordReaderWithSimpleTypes implements JASN1ReadRecordTest
String dataFile = "target/bmpstring_wrapper.dat";
BMPStringWrapper berValue = new BMPStringWrapper();
- berValue.setValue(new BerBMPString("Some UTF-8 String. こんにちは世界。".getBytes()));
+ berValue.setValue(new BerBMPString("Some UTF-8 String. こんにちは世界。".getBytes(StandardCharsets.UTF_8)));
Map<String, Object> expectedValues = new HashMap<String, Object>() {{
put("value", "Some UTF-8 String. こんにちは世界。");
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java
index 523510900c..fe7bde523f 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java
@@ -28,6 +28,7 @@ import org.apache.nifi.serialization.WriteResult;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Map;
@@ -85,7 +86,7 @@ public class MockRecordWriter extends AbstractControllerService implements Recor
@Override
public WriteResult write(final RecordSet rs) throws IOException {
if (header != null && !headerWritten) {
- out.write(header.getBytes());
+ out.write(header.getBytes(StandardCharsets.UTF_8));
out.write("\n".getBytes());
headerWritten = true;
}
@@ -105,10 +106,10 @@ public class MockRecordWriter extends AbstractControllerService implements Recor
if (val != null) {
if (quoteValues) {
out.write("\"".getBytes());
- out.write(val.getBytes());
+ out.write(val.getBytes(StandardCharsets.UTF_8));
out.write("\"".getBytes());
} else {
- out.write(val.getBytes());
+ out.write(val.getBytes(StandardCharsets.UTF_8));
}
}
@@ -134,7 +135,7 @@ public class MockRecordWriter extends AbstractControllerService implements Recor
}
if (header != null && !headerWritten) {
- out.write(header.getBytes());
+ out.write(header.getBytes(StandardCharsets.UTF_8));
out.write("\n".getBytes());
headerWritten = true;
}
diff --git a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/main/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/main/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java
index 2070c781de..9e9183f444 100644
--- a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/main/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java
+++ b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/main/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java
@@ -21,6 +21,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -316,7 +317,7 @@ public class ConvertExcelToCSVProcessor
ff = session.write(ff, new OutputStreamCallback() {
@Override
public void process(OutputStream out) throws IOException {
- PrintStream outPrint = new PrintStream(out);
+ PrintStream outPrint = new PrintStream(out, false, StandardCharsets.UTF_8.name());
sheetHandler.setOutput(outPrint);
try {
@@ -386,7 +387,7 @@ public class ConvertExcelToCSVProcessor
}
public void setOutput(PrintStream output){
- final OutputStreamWriter streamWriter = new OutputStreamWriter(output);
+ final OutputStreamWriter streamWriter = new OutputStreamWriter(output, StandardCharsets.UTF_8);
try {
printer = new CSVPrinter(streamWriter, csvFormat);
diff --git a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/PostSlackTextMessageTest.java b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/PostSlackTextMessageTest.java
index c21f2e6b9d..dc5f379b75 100644
--- a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/PostSlackTextMessageTest.java
+++ b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/PostSlackTextMessageTest.java
@@ -28,6 +28,7 @@ import javax.json.Json;
import javax.json.JsonObject;
import javax.json.JsonReader;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -282,7 +283,8 @@ public class PostSlackTextMessageTest {
private JsonObject getRequestBodyJson() {
try {
final RecordedRequest recordedRequest = mockWebServer.takeRequest();
- try (final JsonReader reader = Json.createReader(new InputStreamReader(recordedRequest.getBody().inputStream()))) {
+ try (final JsonReader reader = Json.createReader(new InputStreamReader(
+ recordedRequest.getBody().inputStream(), StandardCharsets.UTF_8))) {
return reader.readObject();
}
} catch (final InterruptedException e) {