You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2019/08/16 19:29:16 UTC
[metron] 01/02: METRON-614: Eliminate use of the default Charset
(justinleet) closes apache/metron#1341
This is an automated email from the ASF dual-hosted git repository.
leet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
commit f6054a46d6ba56396e61f26eeeb08ec47a40a498
Author: Justin Leet <ju...@gmail.com>
AuthorDate: Fri Aug 16 15:13:18 2019 -0400
METRON-614: Eliminate use of the default Charset (justinleet) closes apache/metron#1341
---
Upgrading.md | 9 +++
.../java/org/apache/metron/maas/util/RESTUtil.java | 4 +-
.../apache/metron/maas/service/runner/Runner.java | 3 +-
.../metron/maas/service/MaasIntegrationTest.java | 7 +-
.../profiler/DefaultMessageDistributorTest.java | 6 +-
.../profiler/hbase/SaltyRowKeyBuilderTest.java | 33 ++++-----
.../profiler/repl/StandAloneProfilerTest.java | 3 +-
.../metron/profiler/storm/ProfileSplitterBolt.java | 11 ++-
.../profiler/storm/ProfileSplitterBoltTest.java | 6 +-
.../storm/integration/ConfigUploadComponent.java | 3 +-
.../storm/integration/ProfilerIntegrationTest.java | 4 +-
.../metron/performance/load/LoadOptions.java | 8 ++-
.../load/monitor/writers/CSVWriter.java | 5 +-
.../metron/performance/load/LoadOptionsTest.java | 5 +-
.../org/apache/metron/rest/config/GrokConfig.java | 3 +-
.../rest/service/impl/DockerStormCLIWrapper.java | 4 +-
.../metron/rest/service/impl/GrokServiceImpl.java | 5 +-
.../impl/SensorEnrichmentConfigServiceImpl.java | 15 ++--
.../impl/SensorIndexingConfigServiceImpl.java | 4 +-
.../impl/SensorParserConfigServiceImpl.java | 6 +-
.../metron/rest/service/impl/StormCLIWrapper.java | 4 +-
.../metron/rest/user/UserSettingsClient.java | 6 +-
.../org/apache/metron/rest/config/TestConfig.java | 29 +++-----
.../controller/GrokControllerIntegrationTest.java | 3 +-
.../controller/HdfsControllerIntegrationTest.java | 3 +-
.../controller/PcapControllerIntegrationTest.java | 5 +-
...rEnrichmentConfigControllerIntegrationTest.java | 24 +++----
...sorIndexingConfigControllerIntegrationTest.java | 9 +--
.../UpdateControllerIntegrationTest.java | 3 +-
.../rest/mock/MockPcapToPdmlScriptWrapper.java | 3 +-
.../service/impl/DockerStormCLIWrapperTest.java | 4 +-
.../service/impl/GlobalConfigServiceImplTest.java | 8 ++-
.../rest/service/impl/GrokServiceImplTest.java | 3 +-
.../rest/service/impl/PcapServiceImplTest.java | 5 +-
.../SensorEnrichmentConfigServiceImplTest.java | 30 ++++----
.../impl/SensorIndexingConfigServiceImplTest.java | 8 ++-
.../impl/SensorParserConfigServiceImplTest.java | 12 ++--
.../metron/rest/user/UserSettingsClientTest.java | 47 ++++++------
.../common/message/metadata/RawMessageUtil.java | 3 +-
.../message/metadata/RawMessageUtilTest.java | 84 +++++++++++++---------
.../common/configuration/ConfigurationsUtils.java | 30 ++++----
.../common/configuration/SensorParserConfig.java | 3 +-
.../enrichment/SensorEnrichmentConfig.java | 3 +-
.../enrichment/SensorEnrichmentUpdateConfig.java | 4 +-
.../configuration/profiler/ProfileConfig.java | 3 +-
.../configuration/profiler/ProfilerConfig.java | 3 +-
.../apache/metron/common/error/MetronError.java | 2 +-
.../metadata/EnvelopedRawMessageStrategy.java | 8 ++-
.../org/apache/metron/common/utils/KafkaUtils.java | 3 +-
.../configurations/ConfigurationsUpdater.java | 7 +-
.../cli/ConfigurationManagerIntegrationTest.java | 9 ++-
.../common/configuration/ConfigurationTest.java | 3 +-
.../configuration/ParserConfigurationsTest.java | 4 +-
.../configuration/SensorEnrichmentConfigTest.java | 10 ++-
.../configuration/SensorParserConfigTest.java | 8 ++-
.../writer/EnrichmentWriterConfigurationTest.java | 3 +-
.../writer/ParserWriterConfigurationTest.java | 4 +-
.../writer/ProfilerWriterConfigurationTest.java | 3 +-
.../metron/common/error/MetronErrorTest.java | 3 +-
.../common/field/FieldNameConvertersTest.java | 3 +-
.../typosquat/TyposquattingStrategiesTest.java | 4 +-
.../apache/metron/common/utils/JSONUtilsTest.java | 7 +-
.../apache/metron/common/utils/KafkaUtilsTest.java | 9 ++-
.../common/utils/file/ReaderSpliteratorTest.java | 5 +-
.../ZKConfigurationsCacheIntegrationTest.java | 15 ++--
.../components/ConfigUploadComponent.java | 18 +++--
.../nonbulk/flatfile/location/RawLocation.java | 11 +--
.../extractor/stix/StixExtractorTest.java | 6 +-
.../metron/dataloads/hbase/mr/HBaseUtil.java | 22 +++---
...pleEnrichmentFlatFileLoaderIntegrationTest.java | 28 +++++---
.../flatfile/SimpleFlatFileSummarizerTest.java | 3 +-
.../bulk/ElasticsearchImportExport.java | 10 ++-
.../enrichment/adapters/cif/CIFHbaseAdapter.java | 5 +-
.../metron/enrichment/cli/LatencySummarizer.java | 4 +-
.../integration/EnrichmentIntegrationTest.java | 21 ++----
.../coprocessor/EnrichmentCoprocessorTest.java | 3 +-
.../metron/hbase/HBaseProjectionCriteria.java | 7 +-
.../org/apache/metron/hbase/mock/MockHTable.java | 3 +-
.../org/apache/metron/indexing/dao/HBaseDao.java | 6 +-
.../integration/HDFSIndexingIntegrationTest.java | 3 +-
.../integration/IndexingIntegrationTest.java | 25 +++----
.../apache/metron/integration/ProcessorResult.java | 3 +-
.../components/FluxTopologyComponent.java | 8 ++-
.../apache/metron/integration/utils/TestUtils.java | 6 +-
.../metron/management/ConfigurationFunctions.java | 11 +--
.../apache/metron/management/GrokFunctions.java | 3 +-
.../metron/management/StellarParserRunner.java | 8 ++-
.../management/ConfigurationFunctionsTest.java | 31 +++++---
.../metron/management/ParserFunctionsTest.java | 14 ++--
.../java/org/apache/metron/parsers/GrokParser.java | 6 +-
.../apache/metron/parsers/ParserRunnerImpl.java | 3 +-
.../org/apache/metron/parsers/csv/CSVParser.java | 5 +-
.../apache/metron/parsers/json/JSONMapParser.java | 5 +-
.../parsers/regex/RegularExpressionsParser.java | 5 +-
.../metron/parsers/syslog/BaseSyslogParser.java | 5 +-
.../metron/parsers/AbstractParserConfigTest.java | 9 ++-
.../org/apache/metron/parsers/GrokParserTest.java | 3 +-
.../apache/metron/parsers/MessageParserTest.java | 13 ++--
.../metron/parsers/MultiLineGrokParserTest.java | 5 +-
.../parsers/MultiLineWithErrorsGrokParserTest.java | 5 +-
.../metron/parsers/ParserRunnerImplTest.java | 23 +++---
.../EnvelopedParserIntegrationTest.java | 16 +++--
.../parsers/integration/ParserIntegrationTest.java | 9 ++-
.../integration/validation/ParserDriver.java | 6 +-
.../validation/SampleDataValidation.java | 5 +-
.../parsers/json/JSONMapParserQueryTest.java | 17 ++---
.../metron/parsers/json/JSONMapParserTest.java | 14 ++--
.../json/JSONMapParserWrappedQueryTest.java | 15 ++--
.../regex/RegularExpressionsParserTest.java | 4 +-
.../parsers/syslog/Syslog3164ParserTest.java | 9 ++-
.../parsers/syslog/Syslog5424ParserTest.java | 16 +++--
.../apache/metron/parsers/asa/BasicAsaParser.java | 12 ++--
.../apache/metron/parsers/bro/BasicBroParser.java | 3 +-
.../org/apache/metron/parsers/cef/CEFParser.java | 4 +-
.../metron/parsers/fireeye/BasicFireEyeParser.java | 2 +-
.../apache/metron/parsers/ise/BasicIseParser.java | 3 +-
.../apache/metron/parsers/ise/JavaCharStream.java | 10 +--
.../metron/parsers/lancope/BasicLancopeParser.java | 3 +-
.../org/apache/metron/parsers/leef/LEEFParser.java | 1 -
.../parsers/logstash/BasicLogstashParser.java | 3 +-
.../paloalto/BasicPaloAltoFirewallParser.java | 3 +-
.../metron/parsers/snort/BasicSnortParser.java | 6 +-
.../parsers/sourcefire/BasicSourcefireParser.java | 3 +-
.../org/apache/metron/parsers/SnortParserTest.java | 9 +--
.../metron/parsers/asa/BasicAsaParserTest.java | 21 +++---
.../metron/parsers/bro/BasicBroParserTest.java | 75 ++++++++++---------
.../apache/metron/parsers/cef/CEFParserTest.java | 55 +++++++-------
.../parsers/fireeye/BasicFireEyeParserTest.java | 5 +-
.../metron/parsers/ise/BasicIseParserTest.java | 3 +-
.../parsers/lancope/BasicLancopeParserTest.java | 3 +-
.../apache/metron/parsers/leef/LEEFParserTest.java | 29 ++++----
.../paloalto/BasicPaloAltoFirewallParserTest.java | 34 ++++-----
.../sourcefire/BasicSourcefireParserTest.java | 5 +-
.../parsers/websphere/GrokWebSphereParserTest.java | 25 ++++---
.../integration/validation/StormParserDriver.java | 27 +++----
.../integration/WriterBoltIntegrationTest.java | 5 +-
.../org/apache/metron/pcap/query/PcapCliTest.java | 8 ++-
.../metron/solr/schema/SchemaTranslator.java | 3 +-
.../metron/test/converters/BinaryConverters.java | 4 +-
.../apache/metron/test/filereaders/FileReader.java | 3 +-
.../test/spouts/GenericInternalTestSpout.java | 3 +-
.../org/apache/metron/test/utils/KafkaLoader.java | 5 +-
.../apache/metron/writer/hdfs/SourceHandler.java | 3 +-
.../writer/bolt/BulkMessageWriterBoltTest.java | 3 +-
.../metron/zookeeper/SimpleEventListener.java | 4 +-
.../stellar/common/shell/cli/StellarShellTest.java | 5 +-
.../stellar/dsl/functions/RestFunctionsTest.java | 9 ++-
pom.xml | 5 +-
148 files changed, 837 insertions(+), 617 deletions(-)
diff --git a/Upgrading.md b/Upgrading.md
index 66fe9c3..66350c3 100644
--- a/Upgrading.md
+++ b/Upgrading.md
@@ -19,6 +19,15 @@ limitations under the License.
This document constitutes a per-version listing of changes of
configuration which are non-backwards compatible.
+
+## 0.7.2 to 0.7.3
+
+### [METRON-614: Eliminate use of the default Charset](https://issues.apache.org/jira/browse/METRON-614)
+The use of the system default Charset is being dropped throughout the code in favor or explicit Charsets, by default UTF-8. As part of this change, individual parsers may now set a configuration property `readCharset` to allow each parser to set the Charset used by its data. This has potential upgrade implications:
+
+* If your system's default Charset and the input data is not UTF-8, you will need to configure each parser to use the appropriate Charset.
+* If you've directly implemented the `MessageParser` interface (instead of extending `BasicParser`) and you need to handle non-UTF-8 Charsets, you should do reading and handling of this configuration in your implementation by overriding `getReadCharset()`.
+
## 0.7.1 to 0.7.2
### [METRON-2164: Remove the Split-Join Enrichment Topology](https://issues.apache.org/jira/browse/METRON-2164)
diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/util/RESTUtil.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/util/RESTUtil.java
index e3b6d8e..8bb6e4e 100644
--- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/util/RESTUtil.java
+++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/util/RESTUtil.java
@@ -19,6 +19,7 @@ package org.apache.metron.maas.util;
import com.google.common.collect.Iterables;
import com.google.common.io.CharStreams;
+import java.nio.charset.StandardCharsets;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
@@ -61,7 +62,8 @@ public enum RESTUtil {
+ response.getStatusLine().getStatusCode());
}
- return new BufferedReader(new InputStreamReader(response.getEntity().getContent()))
+ return new BufferedReader(new InputStreamReader(response.getEntity().getContent(),
+ StandardCharsets.UTF_8))
.lines().collect(Collectors.joining("\n"));
}
public URL appendToUrl(URL endpointUrl, String params) throws MalformedURLException {
diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java
index 4103d3d..e35ab0a 100644
--- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java
+++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java
@@ -21,6 +21,7 @@ import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
import com.google.common.collect.Iterables;
import com.google.common.io.Files;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.cli.*;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
@@ -314,7 +315,7 @@ public class Runner {
}
if(content != null && content.length() > 0) {
try {
- Endpoint ep = ConfigUtil.INSTANCE.read(content.getBytes(), Endpoint.class);
+ Endpoint ep = ConfigUtil.INSTANCE.read(content.getBytes(StandardCharsets.UTF_8), Endpoint.class);
return ep;
}
catch(Exception ex) {
diff --git a/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java
index a75f2a3..8752850 100644
--- a/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java
+++ b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java
@@ -20,6 +20,7 @@ import java.io.*;
import java.net.HttpURLConnection;
import java.net.InetAddress;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
@@ -279,7 +280,7 @@ public class MaasIntegrationTest {
String line;
Process p = Runtime.getRuntime().exec("ps -e");
BufferedReader input =
- new BufferedReader(new InputStreamReader(p.getInputStream()));
+ new BufferedReader(new InputStreamReader(p.getInputStream(), StandardCharsets.UTF_8));
while ((line = input.readLine()) != null) {
if(line.contains("dummy_rest.sh")) {
String pid = Iterables.get(Splitter.on(" ").split(line.replaceAll("\\s+", " ").trim()), 0);
@@ -305,7 +306,7 @@ public class MaasIntegrationTest {
}
BufferedReader br = new BufferedReader(new InputStreamReader(
- (conn.getInputStream())));
+ (conn.getInputStream()), StandardCharsets.UTF_8));
String output = "";
String line;
@@ -407,7 +408,7 @@ public class MaasIntegrationTest {
try {
String sCurrentLine;
- br = new BufferedReader(new FileReader(output));
+ br = new BufferedReader(new InputStreamReader(new FileInputStream(output), StandardCharsets.UTF_8));
int numOfline = 0;
while ((sCurrentLine = br.readLine()) != null) {
if (count) {
diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultMessageDistributorTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultMessageDistributorTest.java
index e04404c..cf11eb7 100644
--- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultMessageDistributorTest.java
+++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultMessageDistributorTest.java
@@ -21,7 +21,6 @@
package org.apache.metron.profiler;
import com.github.benmanes.caffeine.cache.Ticker;
-import com.google.common.util.concurrent.MoreExecutors;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.profiler.ProfileConfig;
import org.apache.metron.common.utils.JSONUtils;
@@ -33,12 +32,9 @@ import org.junit.Test;
import java.io.IOException;
import java.util.List;
-import java.util.Optional;
import java.util.concurrent.TimeUnit;
-import static java.util.concurrent.TimeUnit.HOURS;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.*;
import static org.junit.Assert.assertEquals;
public class DefaultMessageDistributorTest {
diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java
index e92a447..f39659b 100644
--- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java
+++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java
@@ -20,6 +20,7 @@
package org.apache.metron.profiler.hbase;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.profiler.ProfileMeasurement;
import org.apache.metron.profiler.ProfilePeriod;
import org.junit.Assert;
@@ -78,9 +79,9 @@ public class SaltyRowKeyBuilderTest {
ByteBuffer buffer = ByteBuffer
.allocate(100)
.put(SaltyRowKeyBuilder.getSalt(measurement.getPeriod(), saltDivisor))
- .put(measurement.getProfileName().getBytes())
- .put(measurement.getEntity().getBytes())
- .put("group1".getBytes())
+ .put(measurement.getProfileName().getBytes(StandardCharsets.UTF_8))
+ .put(measurement.getEntity().getBytes(StandardCharsets.UTF_8))
+ .put("group1".getBytes(StandardCharsets.UTF_8))
.putLong(1635701L);
buffer.flip();
@@ -104,10 +105,10 @@ public class SaltyRowKeyBuilderTest {
ByteBuffer buffer = ByteBuffer
.allocate(100)
.put(SaltyRowKeyBuilder.getSalt(measurement.getPeriod(), saltDivisor))
- .put(measurement.getProfileName().getBytes())
- .put(measurement.getEntity().getBytes())
- .put("group1".getBytes())
- .put("group2".getBytes())
+ .put(measurement.getProfileName().getBytes(StandardCharsets.UTF_8))
+ .put(measurement.getEntity().getBytes(StandardCharsets.UTF_8))
+ .put("group1".getBytes(StandardCharsets.UTF_8))
+ .put("group2".getBytes(StandardCharsets.UTF_8))
.putLong(1635701L);
buffer.flip();
@@ -131,9 +132,9 @@ public class SaltyRowKeyBuilderTest {
ByteBuffer buffer = ByteBuffer
.allocate(100)
.put(SaltyRowKeyBuilder.getSalt(measurement.getPeriod(), saltDivisor))
- .put(measurement.getProfileName().getBytes())
- .put(measurement.getEntity().getBytes())
- .put("200".getBytes())
+ .put(measurement.getProfileName().getBytes(StandardCharsets.UTF_8))
+ .put(measurement.getEntity().getBytes(StandardCharsets.UTF_8))
+ .put("200".getBytes(StandardCharsets.UTF_8))
.putLong(1635701L);
buffer.flip();
@@ -157,10 +158,10 @@ public class SaltyRowKeyBuilderTest {
ByteBuffer buffer = ByteBuffer
.allocate(100)
.put(SaltyRowKeyBuilder.getSalt(measurement.getPeriod(), saltDivisor))
- .put(measurement.getProfileName().getBytes())
- .put(measurement.getEntity().getBytes())
- .put("200".getBytes())
- .put("group1".getBytes())
+ .put(measurement.getProfileName().getBytes(StandardCharsets.UTF_8))
+ .put(measurement.getEntity().getBytes(StandardCharsets.UTF_8))
+ .put("200".getBytes(StandardCharsets.UTF_8))
+ .put("group1".getBytes(StandardCharsets.UTF_8))
.putLong(1635701L);
buffer.flip();
@@ -184,8 +185,8 @@ public class SaltyRowKeyBuilderTest {
ByteBuffer buffer = ByteBuffer
.allocate(100)
.put(SaltyRowKeyBuilder.getSalt(measurement.getPeriod(), saltDivisor))
- .put(measurement.getProfileName().getBytes())
- .put(measurement.getEntity().getBytes())
+ .put(measurement.getProfileName().getBytes(StandardCharsets.UTF_8))
+ .put(measurement.getEntity().getBytes(StandardCharsets.UTF_8))
.putLong(1635701L);
buffer.flip();
diff --git a/metron-analytics/metron-profiler-repl/src/test/java/org/apache/metron/profiler/repl/StandAloneProfilerTest.java b/metron-analytics/metron-profiler-repl/src/test/java/org/apache/metron/profiler/repl/StandAloneProfilerTest.java
index 94c8fc8..8c63a75 100644
--- a/metron-analytics/metron-profiler-repl/src/test/java/org/apache/metron/profiler/repl/StandAloneProfilerTest.java
+++ b/metron-analytics/metron-profiler-repl/src/test/java/org/apache/metron/profiler/repl/StandAloneProfilerTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.metron.profiler.repl;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.profiler.ProfilerConfig;
import org.apache.metron.common.utils.JSONUtils;
@@ -233,7 +234,7 @@ public class StandAloneProfilerTest {
*/
private ProfilerConfig toProfilerConfig(String configAsJSON) throws Exception {
- InputStream in = new ByteArrayInputStream(configAsJSON.getBytes("UTF-8"));
+ InputStream in = new ByteArrayInputStream(configAsJSON.getBytes(StandardCharsets.UTF_8));
return JSONUtils.INSTANCE.load(in, ProfilerConfig.class);
}
diff --git a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
index 2161910..d28db4a 100644
--- a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
+++ b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
@@ -20,12 +20,12 @@
package org.apache.metron.profiler.storm;
-import org.apache.metron.storm.common.bolt.ConfiguredProfilerBolt;
import org.apache.metron.common.configuration.profiler.ProfilerConfig;
import org.apache.metron.profiler.DefaultMessageRouter;
import org.apache.metron.profiler.MessageRoute;
import org.apache.metron.profiler.MessageRouter;
import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.storm.common.bolt.ConfiguredProfilerBolt;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -40,14 +40,11 @@ import org.slf4j.LoggerFactory;
import java.io.UnsupportedEncodingException;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
-import static org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.OFFSET;
-import static org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.PARTITION;
-import static org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.TIMESTAMP;
-import static org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.TOPIC;
-import static org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.VALUE;
+import static org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.*;
/**
* The Storm bolt responsible for filtering incoming messages and directing
@@ -170,7 +167,7 @@ public class ProfileSplitterBolt extends ConfiguredProfilerBolt {
return;
}
- JSONObject message = (JSONObject) parser.parse(new String(data, "UTF8"));
+ JSONObject message = (JSONObject) parser.parse(new String(data, StandardCharsets.UTF_8));
routeMessage(input, message, config);
}
diff --git a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java
index 360ef4b..c841122 100644
--- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java
+++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java
@@ -20,6 +20,7 @@
package org.apache.metron.profiler.storm;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.profiler.ProfileConfig;
import org.apache.metron.common.configuration.profiler.ProfilerConfig;
@@ -220,7 +221,8 @@ public class ProfileSplitterBoltTest extends BaseBoltTest {
message = (JSONObject) parser.parse(input);
// ensure the tuple returns the expected json message
- when(tuple.getBinaryByField(VALUE.getFieldName())).thenReturn(input.getBytes());
+ when(tuple.getBinaryByField(VALUE.getFieldName())).thenReturn(input.getBytes(
+ StandardCharsets.UTF_8));
}
/**
@@ -430,7 +432,7 @@ public class ProfileSplitterBoltTest extends BaseBoltTest {
* @throws Exception
*/
private ProfilerConfig toProfilerConfig(String configAsJSON) throws Exception {
- InputStream in = new ByteArrayInputStream(configAsJSON.getBytes("UTF-8"));
+ InputStream in = new ByteArrayInputStream(configAsJSON.getBytes(StandardCharsets.UTF_8));
return JSONUtils.INSTANCE.load(in, ProfilerConfig.class);
}
diff --git a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java
index eae3c52..336abd0 100644
--- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java
+++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java
@@ -19,6 +19,7 @@
*/
package org.apache.metron.profiler.storm.integration;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.imps.CuratorFrameworkState;
@@ -96,7 +97,7 @@ public class ConfigUploadComponent implements InMemoryComponent {
configBytes = readProfilerConfigFromFile(profilerConfigurationPath);
} else if(profilerConfig != null) {
- configBytes = profilerConfig.toJSON().getBytes();
+ configBytes = profilerConfig.toJSON().getBytes(StandardCharsets.UTF_8);
}
if (ArrayUtils.getLength(configBytes) > 0) {
diff --git a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
index ea4ad4e..2452102 100644
--- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
+++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
@@ -20,6 +20,7 @@
package org.apache.metron.profiler.storm.integration;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.io.FileUtils;
import org.apache.metron.common.Constants;
@@ -368,7 +369,8 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
}, timeout);
// validate the triage message
- JSONObject message = (JSONObject) new JSONParser().parse(new String(outputMessages.get(0), "UTF-8"));
+ JSONObject message = (JSONObject) new JSONParser().parse(new String(outputMessages.get(0),
+ StandardCharsets.UTF_8));
assertEquals("profile-with-triage", message.get(PROFILE_FIELD));
assertEquals("global", message.get(ENTITY_FIELD));
assertEquals(76548935L, message.get(PERIOD_ID_FIELD));
diff --git a/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/load/LoadOptions.java b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/load/LoadOptions.java
index b4d217d..609f264 100644
--- a/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/load/LoadOptions.java
+++ b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/load/LoadOptions.java
@@ -18,6 +18,9 @@
package org.apache.metron.performance.load;
import com.google.common.base.Joiner;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.cli.*;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.common.utils.cli.CLIOptions;
@@ -122,7 +125,8 @@ public enum LoadOptions implements CLIOptions<LoadOptions> {
}
File discreteDistributionFile = new File(option.get(cli));
if(discreteDistributionFile.exists()) {
- try (BufferedReader br = new BufferedReader(new FileReader(discreteDistributionFile))){
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(discreteDistributionFile),
+ StandardCharsets.UTF_8))){
return Optional.ofNullable(BiasedSampler.readDistribution(br));
} catch (IOException e) {
throw new IllegalStateException("Unable to read distribution file: " + option.get(cli), e);
@@ -180,7 +184,7 @@ public enum LoadOptions implements CLIOptions<LoadOptions> {
File templateFile = new File(option.get(cli));
if(templateFile.exists()) {
List<String> templates = new ArrayList<>();
- try(BufferedReader br = new BufferedReader(new FileReader(templateFile))) {
+ try(BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(templateFile), StandardCharsets.UTF_8))) {
for(String line = null;(line = br.readLine()) != null;) {
templates.add(line);
}
diff --git a/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/load/monitor/writers/CSVWriter.java b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/load/monitor/writers/CSVWriter.java
index 112206d..34df517 100644
--- a/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/load/monitor/writers/CSVWriter.java
+++ b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/load/monitor/writers/CSVWriter.java
@@ -18,6 +18,9 @@
package org.apache.metron.performance.load.monitor.writers;
import com.google.common.base.Joiner;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.performance.load.monitor.Results;
import java.io.File;
@@ -34,7 +37,7 @@ public class CSVWriter implements Consumer<Writable> {
public CSVWriter(File outFile) throws IOException {
if(outFile != null) {
- pw = Optional.of(new PrintWriter(new FileWriter(outFile)));
+ pw = Optional.of(new PrintWriter(new OutputStreamWriter(new FileOutputStream(outFile), StandardCharsets.UTF_8)));
}
}
diff --git a/metron-contrib/metron-performance/src/test/java/org/apache/metron/performance/load/LoadOptionsTest.java b/metron-contrib/metron-performance/src/test/java/org/apache/metron/performance/load/LoadOptionsTest.java
index 0d9b34a..c8a6f6e 100644
--- a/metron-contrib/metron-performance/src/test/java/org/apache/metron/performance/load/LoadOptionsTest.java
+++ b/metron-contrib/metron-performance/src/test/java/org/apache/metron/performance/load/LoadOptionsTest.java
@@ -17,6 +17,9 @@
*/
package org.apache.metron.performance.load;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.io.IOUtils;
@@ -75,7 +78,7 @@ public class LoadOptionsTest {
public void testTemplatePresent() throws Exception {
File templateFile= new File("target/template");
String template = "test template1";
- try(BufferedWriter w = new BufferedWriter(new FileWriter(templateFile))) {
+ try(BufferedWriter w = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(templateFile), StandardCharsets.UTF_8))) {
IOUtils.write(template, w );
}
templateFile.deleteOnExit();
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java
index 6b0bc1b..2b2db47 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.config;
+import java.nio.charset.StandardCharsets;
import oi.thekraken.grok.api.Grok;
import oi.thekraken.grok.api.exception.GrokException;
import org.springframework.context.annotation.Bean;
@@ -31,7 +32,7 @@ public class GrokConfig {
public Grok commonGrok() throws GrokException {
Grok grok = new Grok();
grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream(
- "/patterns/common")));
+ "/patterns/common"), StandardCharsets.UTF_8));
return grok;
}
}
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java
index 059afd7..168d11d 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.service.impl;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.lang3.ArrayUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,7 +54,8 @@ public class DockerStormCLIWrapper extends StormCLIWrapper {
final ProcessBuilder pb = getDockerEnvironmentProcessBuilder();
try {
final Process process = pb.start();
- final BufferedReader inputStream = new BufferedReader(new InputStreamReader(process.getInputStream()));
+ final BufferedReader inputStream = new BufferedReader(new InputStreamReader(process.getInputStream(),
+ StandardCharsets.UTF_8));
String line;
while ((line = inputStream.readLine()) != null) {
if (line.startsWith("export")) {
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
index 11ab664..c43c661 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
@@ -18,6 +18,7 @@
package org.apache.metron.rest.service.impl;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import oi.thekraken.grok.api.Grok;
import oi.thekraken.grok.api.Match;
import org.apache.commons.io.IOUtils;
@@ -72,7 +73,7 @@ public class GrokServiceImpl implements GrokService {
}
Grok grok = new Grok();
grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream(
- "/patterns/common")));
+ "/patterns/common"), StandardCharsets.UTF_8));
grok.addPatternFromReader(new StringReader(grokValidation.getStatement()));
String grokPattern = "%{" + grokValidation.getPatternLabel() + "}";
grok.compile(grokPattern);
@@ -92,7 +93,7 @@ public class GrokServiceImpl implements GrokService {
if (statement != null) {
Path path = getTemporaryGrokRootPath();
hdfsService.mkdirs(path);
- hdfsService.write(new Path(path, name), statement.getBytes(Charset.forName("utf-8")),null,null,null);
+ hdfsService.write(new Path(path, name), statement.getBytes(StandardCharsets.UTF_8),null,null,null);
return path;
} else {
throw new RestException("A grokStatement must be provided");
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
index 5c0f2e0..055d990 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
@@ -18,13 +18,6 @@
package org.apache.metron.rest.service.impl;
import com.fasterxml.jackson.databind.ObjectMapper;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
import org.apache.curator.framework.CuratorFramework;
import org.apache.metron.common.aggregator.Aggregators;
import org.apache.metron.common.configuration.ConfigurationType;
@@ -39,6 +32,11 @@ import org.apache.zookeeper.KeeperException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.stream.Collectors;
+
@Service
public class SensorEnrichmentConfigServiceImpl implements SensorEnrichmentConfigService {
@@ -63,7 +61,8 @@ public class SensorEnrichmentConfigServiceImpl implements SensorEnrichmentConfig
@Override
public SensorEnrichmentConfig save(String name, SensorEnrichmentConfig sensorEnrichmentConfig) throws RestException {
try {
- ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(name, objectMapper.writeValueAsString(sensorEnrichmentConfig).getBytes(), client);
+ ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(name, objectMapper.writeValueAsString(sensorEnrichmentConfig).getBytes(
+ StandardCharsets.UTF_8), client);
} catch (Exception e) {
throw new RestException(e);
}
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
index a90d123..75cb918 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
@@ -19,6 +19,7 @@ package org.apache.metron.rest.service.impl;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Iterables;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.metron.common.configuration.ConfigurationType;
@@ -58,7 +59,8 @@ public class SensorIndexingConfigServiceImpl implements SensorIndexingConfigServ
@Override
public Map<String, Object> save(String name, Map<String, Object> sensorIndexingConfig) throws RestException {
try {
- ConfigurationsUtils.writeSensorIndexingConfigToZookeeper(name, objectMapper.writeValueAsString(sensorIndexingConfig).getBytes(), client);
+ ConfigurationsUtils.writeSensorIndexingConfigToZookeeper(name, objectMapper.writeValueAsString(sensorIndexingConfig).getBytes(
+ StandardCharsets.UTF_8), client);
} catch (Exception e) {
throw new RestException(e);
}
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index d0e4b3d..624f3a8 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -20,6 +20,7 @@ package org.apache.metron.rest.service.impl;
import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
import com.fasterxml.jackson.databind.ObjectMapper;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -71,7 +72,7 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
public SensorParserConfig save(String name, SensorParserConfig sensorParserConfig) throws RestException {
try {
ConfigurationsUtils.writeSensorParserConfigToZookeeper(name,
- objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client);
+ objectMapper.writeValueAsString(sensorParserConfig).getBytes(StandardCharsets.UTF_8), client);
} catch (Exception e) {
throw new RestException(e);
}
@@ -152,7 +153,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
parser.configure(sensorParserConfig.getParserConfig());
parser.init();
- Optional<MessageParserResult<JSONObject>> result = parser.parseOptionalResult(parseMessageRequest.getSampleData().getBytes());
+ Optional<MessageParserResult<JSONObject>> result = parser.parseOptionalResult(parseMessageRequest.getSampleData().getBytes(
+ StandardCharsets.UTF_8));
if (!result.isPresent()) {
throw new RestException("Unknown error parsing sample data");
}
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
index 86e3512..abbc67f 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
@@ -24,6 +24,7 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -180,7 +181,8 @@ public class StormCLIWrapper {
} catch (IOException e) {
throw new RestException(e);
}
- BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream(),
+ StandardCharsets.UTF_8));
List<String> lines = reader.lines().collect(toList());
lines.forEach(System.out::println);
if (lines.size() > 1) {
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/user/UserSettingsClient.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/user/UserSettingsClient.java
index a08b775..7fa0f37 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/user/UserSettingsClient.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/user/UserSettingsClient.java
@@ -76,7 +76,7 @@ public class UserSettingsClient {
}
try {
userSettingsTable = tableProvider.getTable(HBaseConfiguration.create(), table);
- this.cf = cf.getBytes();
+ this.cf = cf.getBytes(StandardCharsets.UTF_8);
} catch (IOException e) {
throw new IllegalStateException("Unable to initialize HBaseDao: " + e.getMessage(), e);
}
@@ -106,7 +106,7 @@ public class UserSettingsClient {
ResultScanner results = getTableInterface().getScanner(scan);
Map<String, Map<String, String>> allUserSettings = new HashMap<>();
for (Result result : results) {
- allUserSettings.put(new String(result.getRow()), getAllUserSettings(result));
+ allUserSettings.put(new String(result.getRow(), StandardCharsets.UTF_8), getAllUserSettings(result));
}
return allUserSettings;
}
@@ -116,7 +116,7 @@ public class UserSettingsClient {
ResultScanner results = getTableInterface().getScanner(scan);
Map<String, Optional<String>> allUserSettings = new HashMap<>();
for (Result result : results) {
- allUserSettings.put(new String(result.getRow()), getUserSettings(result, type));
+ allUserSettings.put(new String(result.getRow(), StandardCharsets.UTF_8), getUserSettings(result, type));
}
return allUserSettings;
}
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
index d42f128..c21068a 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
@@ -17,18 +17,6 @@
*/
package org.apache.metron.rest.config;
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
import kafka.admin.AdminUtils$;
import kafka.utils.ZKStringSerializer$;
import kafka.utils.ZkUtils;
@@ -55,11 +43,7 @@ import org.apache.metron.integration.components.ZKServerComponent;
import org.apache.metron.job.manager.InMemoryJobManager;
import org.apache.metron.job.manager.JobManager;
import org.apache.metron.rest.RestException;
-import org.apache.metron.rest.mock.MockPcapJob;
-import org.apache.metron.rest.mock.MockPcapJobSupplier;
-import org.apache.metron.rest.mock.MockPcapToPdmlScriptWrapper;
-import org.apache.metron.rest.mock.MockStormCLIClientWrapper;
-import org.apache.metron.rest.mock.MockStormRestTemplate;
+import org.apache.metron.rest.mock.*;
import org.apache.metron.rest.service.StormStatusService;
import org.apache.metron.rest.service.impl.CachedStormStatusServiceImpl;
import org.apache.metron.rest.service.impl.PcapToPdmlScriptWrapper;
@@ -74,6 +58,12 @@ import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.web.client.RestTemplate;
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+
@Configuration
@Profile(TEST_PROFILE)
public class TestConfig {
@@ -116,9 +106,10 @@ public class TestConfig {
try {
runner.start();
File globalConfigFile = new File("src/test/resources/zookeeper/global.json");
- try(BufferedReader r = new BufferedReader(new FileReader(globalConfigFile))){
+ try(BufferedReader r = new BufferedReader(new InputStreamReader(new FileInputStream(globalConfigFile), StandardCharsets.UTF_8))){
String globalConfig = IOUtils.toString(r);
- ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig.getBytes(), zkServerComponent.getConnectionString());
+ ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig.getBytes(
+ StandardCharsets.UTF_8), zkServerComponent.getConnectionString());
} catch (Exception e) {
throw new IllegalStateException("Unable to upload global config", e);
}
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
index b2f98fb..ee7587e 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.controller;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.io.FileUtils;
import org.junit.Before;
@@ -135,7 +136,7 @@ public class GrokControllerIntegrationTest {
this.mockMvc.perform(get(grokUrl + "/get/statement?path=/patterns/squid").with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")))
- .andExpect(content().bytes(statement.getBytes()));
+ .andExpect(content().bytes(statement.getBytes(StandardCharsets.UTF_8)));
this.mockMvc.perform(get(grokUrl + "/get/statement?path=/bad/path").with(httpBasic(user,password)))
.andExpect(status().isInternalServerError())
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java
index a6c3469..a5c6043 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.controller;
+import java.nio.charset.StandardCharsets;
import org.apache.hadoop.fs.Path;
import org.apache.metron.rest.service.HdfsService;
import org.junit.Before;
@@ -90,7 +91,7 @@ public class HdfsControllerIntegrationTest {
this.mockMvc.perform(get(hdfsUrl + "?path=" + path).with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")))
- .andExpect(content().bytes(fileContents.getBytes()));
+ .andExpect(content().bytes(fileContents.getBytes(StandardCharsets.UTF_8)));
this.mockMvc.perform(delete(hdfsUrl + "?path=" + path).with(httpBasic(user,password)).with(csrf()))
.andExpect(status().isOk());
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
index 2d7c505..737f8bb 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.controller;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.Path;
@@ -407,14 +408,14 @@ public class PcapControllerIntegrationTest {
.andExpect(header().string("Content-Disposition", "attachment; filename=\"pcap_jobId_1.pcap\""))
.andExpect(header().string("Content-Length", Integer.toString(pcapFileContents.length())))
.andExpect(content().contentType(MediaType.parseMediaType("application/octet-stream")))
- .andExpect(content().bytes(pcapFileContents.getBytes()));
+ .andExpect(content().bytes(pcapFileContents.getBytes(StandardCharsets.UTF_8)));
this.mockMvc.perform(get(pcapUrl + "/jobId/raw?page=1&fileName=pcapFile.pcap").with(httpBasic(user, password)))
.andExpect(status().isOk())
.andExpect(header().string("Content-Disposition", "attachment; filename=\"pcapFile.pcap\""))
.andExpect(header().string("Content-Length", Integer.toString(pcapFileContents.length())))
.andExpect(content().contentType(MediaType.parseMediaType("application/octet-stream")))
- .andExpect(content().bytes(pcapFileContents.getBytes()));
+ .andExpect(content().bytes(pcapFileContents.getBytes(StandardCharsets.UTF_8)));
this.mockMvc.perform(get(pcapUrl + "/jobId/raw?page=2").with(httpBasic(user, password)))
.andExpect(status().isNotFound());
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
index d77e3ae..d25ee04 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
@@ -17,18 +17,6 @@
*/
package org.apache.metron.rest.controller;
-import static org.apache.metron.integration.utils.TestUtils.assertEventually;
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
-import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
-
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.rest.service.SensorEnrichmentConfigService;
import org.hamcrest.core.IsCollectionContaining;
@@ -44,6 +32,16 @@ import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
+import java.nio.charset.StandardCharsets;
+
+import static org.apache.metron.integration.utils.TestUtils.assertEventually;
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
+
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
@ActiveProfiles(TEST_PROFILE)
@@ -152,7 +150,7 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
this.mockMvc.perform(get(sensorEnrichmentConfigUrl).with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(content().bytes("{}".getBytes()));
+ .andExpect(content().bytes("{}".getBytes(StandardCharsets.UTF_8)));
this.mockMvc.perform(post(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
.andExpect(status().isCreated())
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
index d78bedf..cde71fd 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
@@ -29,6 +29,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.rest.service.SensorIndexingConfigService;
import org.junit.Before;
@@ -99,17 +100,17 @@ public class SensorIndexingConfigControllerIntegrationTest {
this.mockMvc.perform(get(sensorIndexingConfigUrl).with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(content().bytes("{}".getBytes()));
+ .andExpect(content().bytes("{}".getBytes(StandardCharsets.UTF_8)));
this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/elasticsearch").with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(content().bytes("[]".getBytes()));
+ .andExpect(content().bytes("[]".getBytes(StandardCharsets.UTF_8)));
this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/blah").with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(content().bytes("[]".getBytes()));
+ .andExpect(content().bytes("[]".getBytes(StandardCharsets.UTF_8)));
this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
.andExpect(status().isCreated())
@@ -120,7 +121,7 @@ public class SensorIndexingConfigControllerIntegrationTest {
assertEventually(() -> this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/elasticsearch").with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(content().bytes("[\"broTest\"]".getBytes()))
+ .andExpect(content().bytes("[\"broTest\"]".getBytes(StandardCharsets.UTF_8)))
);
assertEventually(() -> this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
index 7995557..02f3f3a 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
@@ -18,6 +18,7 @@
package org.apache.metron.rest.controller;
import com.google.common.collect.ImmutableMap;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.hbase.client.Get;
@@ -185,7 +186,7 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest {
//ensure hbase is up to date
Get g = new Get(new HBaseDao.Key(guid,"bro").toBytes());
Result r = table.get(g);
- NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes());
+ NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(1, columns.size());
}
}
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapToPdmlScriptWrapper.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapToPdmlScriptWrapper.java
index 940648c..498ae94 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapToPdmlScriptWrapper.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapToPdmlScriptWrapper.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.mock;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -50,6 +51,6 @@ public class MockPcapToPdmlScriptWrapper extends PcapToPdmlScriptWrapper {
@Override
public InputStream execute(String scriptPath, FileSystem fileSystem, Path pcapPath) throws IOException {
- return new ByteArrayInputStream(pdmlXml.getBytes());
+ return new ByteArrayInputStream(pdmlXml.getBytes(StandardCharsets.UTF_8));
}
}
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
index bc91b91..d2847f0 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.service.impl;
+import java.nio.charset.StandardCharsets;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -60,7 +61,8 @@ public class DockerStormCLIWrapperTest {
when(processBuilder.command()).thenReturn(new ArrayList<>());
Process process = mock(Process.class);
- InputStream inputStream = new ByteArrayInputStream("export DOCKER_HOST=\"tcp://192.168.99.100:2376\"".getBytes());
+ InputStream inputStream = new ByteArrayInputStream("export DOCKER_HOST=\"tcp://192.168.99.100:2376\"".getBytes(
+ StandardCharsets.UTF_8));
when(processBuilder.start()).thenReturn(process);
when(process.getInputStream()).thenReturn(inputStream);
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java
index 85a66b3..caddf8b 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java
@@ -26,6 +26,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
@@ -133,7 +134,8 @@ public class GlobalConfigServiceImplTest {
exception.expect(RestException.class);
SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
- when(setDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot(), "{ }".getBytes())).thenThrow(Exception.class);
+ when(setDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot(), "{ }".getBytes(
+ StandardCharsets.UTF_8))).thenThrow(Exception.class);
when(curatorFramework.setData()).thenReturn(setDataBuilder);
@@ -143,11 +145,11 @@ public class GlobalConfigServiceImplTest {
@Test
public void saveShouldReturnSameConfigThatIsPassedOnSuccessfulSave() throws Exception {
SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
- when(setDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot(), "{ }".getBytes())).thenReturn(new Stat());
+ when(setDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot(), "{ }".getBytes(StandardCharsets.UTF_8))).thenReturn(new Stat());
when(curatorFramework.setData()).thenReturn(setDataBuilder);
assertEquals(new HashMap<>(), globalConfigService.save(new HashMap<>()));
- verify(setDataBuilder).forPath(eq(ConfigurationType.GLOBAL.getZookeeperRoot()), eq("{ }".getBytes()));
+ verify(setDataBuilder).forPath(eq(ConfigurationType.GLOBAL.getZookeeperRoot()), eq("{ }".getBytes(StandardCharsets.UTF_8)));
}
}
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
index f308ba3..4e06b59 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
@@ -25,6 +25,7 @@ import static org.powermock.api.mockito.PowerMockito.when;
import java.io.File;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import oi.thekraken.grok.api.Grok;
@@ -201,7 +202,7 @@ public class GrokServiceImplTest {
grokService.saveTemporary(statement, "squid");
File testRoot = new File("./target/user1");
- assertEquals(statement, FileUtils.readFileToString(new File(testRoot, "squid"), Charset.forName("utf-8")));
+ assertEquals(statement, FileUtils.readFileToString(new File(testRoot, "squid"), StandardCharsets.UTF_8));
testRoot.delete();
}
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
index 8cd363a..62b197b 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
@@ -34,6 +34,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@@ -547,13 +548,13 @@ public class PcapServiceImplTest {
doReturn(fileSystem).when(pcapService).getFileSystem();
when(fileSystem.exists(path)).thenReturn(true);
doReturn(path).when(pcapService).getPath("user", "jobId", 1);
- doReturn(new ByteArrayInputStream(pdmlXml.getBytes())).when(pcapToPdmlScriptWrapper).getRawInputStream(fileSystem, path);
+ doReturn(new ByteArrayInputStream(pdmlXml.getBytes(StandardCharsets.UTF_8))).when(pcapToPdmlScriptWrapper).getRawInputStream(fileSystem, path);
ProcessBuilder pb = PowerMockito.mock(ProcessBuilder.class);
Process p = PowerMockito.mock(Process.class);
OutputStream outputStream = new ByteArrayOutputStream();
when(p.getOutputStream()).thenReturn(outputStream);
when(p.isAlive()).thenReturn(true);
- when(p.getInputStream()).thenReturn(new ByteArrayInputStream(pdmlXml.getBytes()));
+ when(p.getInputStream()).thenReturn(new ByteArrayInputStream(pdmlXml.getBytes(StandardCharsets.UTF_8)));
whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(pb);
PowerMockito.when(pb.start()).thenReturn(p);
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java
index 3d07da5..5cc9705 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java
@@ -17,21 +17,8 @@
*/
package org.apache.metron.rest.service.impl;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.DeleteBuilder;
@@ -52,6 +39,16 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.*;
+
@SuppressWarnings("ALL")
public class SensorEnrichmentConfigServiceImplTest {
@Rule
@@ -189,7 +186,8 @@ public class SensorEnrichmentConfigServiceImplTest {
exception.expect(RestException.class);
SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
- when(setDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro", broJson.getBytes())).thenThrow(Exception.class);
+ when(setDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro", broJson.getBytes(
+ StandardCharsets.UTF_8))).thenThrow(Exception.class);
when(curatorFramework.setData()).thenReturn(setDataBuilder);
@@ -203,11 +201,11 @@ public class SensorEnrichmentConfigServiceImplTest {
when(objectMapper.writeValueAsString(sensorEnrichmentConfig)).thenReturn(broJson);
SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
- when(setDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro", broJson.getBytes())).thenReturn(new Stat());
+ when(setDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro", broJson.getBytes(StandardCharsets.UTF_8))).thenReturn(new Stat());
when(curatorFramework.setData()).thenReturn(setDataBuilder);
assertEquals(sensorEnrichmentConfig, sensorEnrichmentConfigService.save("bro", sensorEnrichmentConfig));
- verify(setDataBuilder).forPath(eq(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro"), eq(broJson.getBytes()));
+ verify(setDataBuilder).forPath(eq(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro"), eq(broJson.getBytes(StandardCharsets.UTF_8)));
}
@Test
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
index 1d8e3f5..dcdd41d 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
@@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.DeleteBuilder;
@@ -262,7 +263,8 @@ public class SensorIndexingConfigServiceImplTest {
exception.expect(RestException.class);
SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
- when(setDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro", broJson.getBytes())).thenThrow(Exception.class);
+ when(setDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro", broJson.getBytes(
+ StandardCharsets.UTF_8))).thenThrow(Exception.class);
when(curatorFramework.setData()).thenReturn(setDataBuilder);
@@ -276,11 +278,11 @@ public class SensorIndexingConfigServiceImplTest {
when(objectMapper.writeValueAsString(sensorIndexingConfig)).thenReturn(broJson);
SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
- when(setDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro", broJson.getBytes())).thenReturn(new Stat());
+ when(setDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro", broJson.getBytes(StandardCharsets.UTF_8))).thenReturn(new Stat());
when(curatorFramework.setData()).thenReturn(setDataBuilder);
assertEquals(sensorIndexingConfig, sensorIndexingConfigService.save("bro", sensorIndexingConfig));
- verify(setDataBuilder).forPath(eq(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro"), eq(broJson.getBytes()));
+ verify(setDataBuilder).forPath(eq(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro"), eq(broJson.getBytes(StandardCharsets.UTF_8)));
}
private Map<String, Object> getTestSensorIndexingConfig() {
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
index 3e54e07..a588b99 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
@@ -19,6 +19,10 @@ package org.apache.metron.rest.service.impl;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import oi.thekraken.grok.api.Grok;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
@@ -214,7 +218,7 @@ public class SensorParserConfigServiceImplTest {
exception.expect(RestException.class);
SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
- when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro", broJson.getBytes())).thenThrow(Exception.class);
+ when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro", broJson.getBytes(StandardCharsets.UTF_8))).thenThrow(Exception.class);
when(curatorFramework.setData()).thenReturn(setDataBuilder);
@@ -230,11 +234,11 @@ public class SensorParserConfigServiceImplTest {
when(objectMapper.writeValueAsString(sensorParserConfig)).thenReturn(broJson);
SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
- when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro", broJson.getBytes())).thenReturn(new Stat());
+ when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro", broJson.getBytes(StandardCharsets.UTF_8))).thenReturn(new Stat());
when(curatorFramework.setData()).thenReturn(setDataBuilder);
assertEquals(getTestBroSensorParserConfig(), sensorParserConfigService.save("bro", sensorParserConfig));
- verify(setDataBuilder).forPath(eq(ConfigurationType.PARSER.getZookeeperRoot() + "/bro"), eq(broJson.getBytes()));
+ verify(setDataBuilder).forPath(eq(ConfigurationType.PARSER.getZookeeperRoot() + "/bro"), eq(broJson.getBytes(StandardCharsets.UTF_8)));
}
@Test
@@ -258,7 +262,7 @@ public class SensorParserConfigServiceImplTest {
File grokRoot = new File("./target", user);
grokRoot.mkdir();
File patternFile = new File(grokRoot, "squid");
- FileWriter writer = new FileWriter(patternFile);
+ Writer writer = new OutputStreamWriter(new FileOutputStream(patternFile), StandardCharsets.UTF_8);
writer.write(grokStatement);
writer.close();
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/user/UserSettingsClientTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/user/UserSettingsClientTest.java
index 5b4f786..f856bcf 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/user/UserSettingsClientTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/user/UserSettingsClientTest.java
@@ -18,31 +18,29 @@
package org.apache.metron.rest.user;
-import static org.apache.metron.rest.user.UserSettingsClient.USER_SETTINGS_HBASE_CF;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.ObjectMapper;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.function.Supplier;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Supplier;
+
+import static org.apache.metron.rest.user.UserSettingsClient.USER_SETTINGS_HBASE_CF;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
public class UserSettingsClientTest {
@Rule
@@ -67,8 +65,9 @@ public class UserSettingsClientTest {
@Test
public void shouldFindOne() throws Exception {
Result result = mock(Result.class);
- when(result.getValue(cf, Bytes.toBytes("type"))).thenReturn("userSettings1String".getBytes());
- Get get = new Get("user1".getBytes());
+ when(result.getValue(cf, Bytes.toBytes("type"))).thenReturn("userSettings1String".getBytes(
+ StandardCharsets.UTF_8));
+ Get get = new Get("user1".getBytes(StandardCharsets.UTF_8));
get.addFamily(cf);
when(userSettingsTable.get(get)).thenReturn(result);
@@ -82,10 +81,12 @@ public class UserSettingsClientTest {
ResultScanner resultScanner = mock(ResultScanner.class);
Result result1 = mock(Result.class);
Result result2 = mock(Result.class);
- when(result1.getRow()).thenReturn("user1".getBytes());
- when(result2.getRow()).thenReturn("user2".getBytes());
- when(result1.getValue(cf, Bytes.toBytes("type"))).thenReturn("userSettings1String".getBytes());
- when(result2.getValue(cf, Bytes.toBytes("type"))).thenReturn("userSettings2String".getBytes());
+ when(result1.getRow()).thenReturn("user1".getBytes(StandardCharsets.UTF_8));
+ when(result2.getRow()).thenReturn("user2".getBytes(StandardCharsets.UTF_8));
+ when(result1.getValue(cf, Bytes.toBytes("type"))).thenReturn("userSettings1String".getBytes(
+ StandardCharsets.UTF_8));
+ when(result2.getValue(cf, Bytes.toBytes("type"))).thenReturn("userSettings2String".getBytes(
+ StandardCharsets.UTF_8));
when(resultScanner.iterator()).thenReturn(Arrays.asList(result1, result2).iterator());
when(userSettingsTable.getScanner(any(Scan.class))).thenReturn(resultScanner);
diff --git a/metron-platform/metron-common-streaming/metron-common-storm/src/main/java/org/apache/metron/storm/common/message/metadata/RawMessageUtil.java b/metron-platform/metron-common-streaming/metron-common-storm/src/main/java/org/apache/metron/storm/common/message/metadata/RawMessageUtil.java
index afd3a87..e9e8c72 100644
--- a/metron-platform/metron-common-streaming/metron-common-storm/src/main/java/org/apache/metron/storm/common/message/metadata/RawMessageUtil.java
+++ b/metron-platform/metron-common-streaming/metron-common-storm/src/main/java/org/apache/metron/storm/common/message/metadata/RawMessageUtil.java
@@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
@@ -88,7 +89,7 @@ public enum RawMessageUtil {
byte[] keyObj = t.getBinary(KEY_INDEX);
String keyStr = null;
try {
- keyStr = keyObj == null ? null : new String(keyObj);
+ keyStr = keyObj == null ? null : new String(keyObj, StandardCharsets.UTF_8);
if (!StringUtils.isEmpty(keyStr)) {
Map<String, Object> rawMetadata = JSONUtils.INSTANCE.load(keyStr, JSONUtils.MAP_SUPPLIER);
for (Map.Entry<String, Object> kv : rawMetadata.entrySet()) {
diff --git a/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/message/metadata/RawMessageUtilTest.java b/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/message/metadata/RawMessageUtilTest.java
index cf51f02..4abbcc0 100644
--- a/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/message/metadata/RawMessageUtilTest.java
+++ b/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/message/metadata/RawMessageUtilTest.java
@@ -18,6 +18,7 @@
package org.apache.metron.storm.common.message.metadata;
import com.google.common.collect.ImmutableMap;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.Constants;
import org.apache.metron.common.message.metadata.EnvelopedRawMessageStrategy;
@@ -57,7 +58,8 @@ public class RawMessageUtilTest {
}
when(t.getFields()).thenReturn(f);
- when(t.getBinary(eq(RawMessageUtil.KEY_INDEX))).thenReturn(metadata.getBytes());
+ when(t.getBinary(eq(RawMessageUtil.KEY_INDEX))).thenReturn(metadata.getBytes(
+ StandardCharsets.UTF_8));
return t;
}
@@ -99,14 +101,16 @@ public class RawMessageUtilTest {
Tuple t = createTuple( kafkaMetadata
, appMetadata);
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(), true, new HashMap<>());
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(
+ StandardCharsets.UTF_8), true, new HashMap<>());
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
checkKafkaMetadata(m, false);
checkAppMetadata(m, false);
}
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(), false, new HashMap<>());
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(
+ StandardCharsets.UTF_8), false, new HashMap<>());
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
Assert.assertTrue(m.getMetadata().isEmpty());
}
}
@@ -116,14 +120,16 @@ public class RawMessageUtilTest {
Tuple t = createTuple(kafkaMetadata
,"{}");
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(), true, new HashMap<>());
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(
+ StandardCharsets.UTF_8), true, new HashMap<>());
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
checkKafkaMetadata(m, false);
checkAppMetadata(m, true);
}
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(), false, new HashMap<>());
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(
+ StandardCharsets.UTF_8), false, new HashMap<>());
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
Assert.assertTrue(m.getMetadata().isEmpty());
}
}
@@ -132,14 +138,16 @@ public class RawMessageUtilTest {
public void testDefaultStrategy_withoutKafkaMetadata_withAppMetadata() throws Exception {
Tuple t = createTuple(new HashMap<>() ,appMetadata);
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(), true, new HashMap<>());
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(
+ StandardCharsets.UTF_8), true, new HashMap<>());
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
checkKafkaMetadata(m, true);
checkAppMetadata(m, false);
}
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(), false, new HashMap<>());
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(
+ StandardCharsets.UTF_8), false, new HashMap<>());
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
Assert.assertTrue(m.getMetadata().isEmpty());
}
}
@@ -148,14 +156,16 @@ public class RawMessageUtilTest {
public void testDefaultStrategy_withoutKafkaMetadata_withoutAppMetadata() throws Exception {
Tuple t = createTuple(new HashMap<>() , "{}");
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(), true, new HashMap<>());
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(
+ StandardCharsets.UTF_8), true, new HashMap<>());
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
checkKafkaMetadata(m, true);
checkAppMetadata(m, true);
}
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(), false, new HashMap<>());
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage(RawMessageStrategies.DEFAULT, t, "raw_message".getBytes(
+ StandardCharsets.UTF_8), false, new HashMap<>());
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
Assert.assertTrue(m.getMetadata().isEmpty());
}
}
@@ -197,8 +207,9 @@ public class RawMessageUtilTest {
, appMetadata);
Map<String, Object> config = ImmutableMap.of(EnvelopedRawMessageStrategy.MESSAGE_FIELD_CONFIG, "data");
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(), true, config);
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(
+ StandardCharsets.UTF_8), true, config);
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
Assert.assertFalse(m.getMetadata().containsKey("data"));
checkEnvelopeMetadata(m);
checkMergedData(m);
@@ -207,9 +218,10 @@ public class RawMessageUtilTest {
}
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(), false, config);
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(
+ StandardCharsets.UTF_8), false, config);
checkMergedData(m);
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
Assert.assertFalse(m.getMetadata().containsKey("data"));
Assert.assertTrue(m.getMetadata().isEmpty());
}
@@ -221,8 +233,9 @@ public class RawMessageUtilTest {
,"{}");
Map<String, Object> config = ImmutableMap.of(EnvelopedRawMessageStrategy.MESSAGE_FIELD_CONFIG, "data");
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(), true, config);
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(
+ StandardCharsets.UTF_8), true, config);
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
Assert.assertFalse(m.getMetadata().containsKey("data"));
checkMergedData(m);
checkEnvelopeMetadata(m);
@@ -230,10 +243,11 @@ public class RawMessageUtilTest {
checkAppMetadata(m, true);
}
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(), false, config);
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(
+ StandardCharsets.UTF_8), false, config);
Assert.assertFalse(m.getMetadata().containsKey("data"));
checkMergedData(m);
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
Assert.assertTrue(m.getMetadata().isEmpty());
}
}
@@ -243,19 +257,21 @@ public class RawMessageUtilTest {
Tuple t = createTuple(new HashMap<>() ,appMetadata);
Map<String, Object> config = ImmutableMap.of(EnvelopedRawMessageStrategy.MESSAGE_FIELD_CONFIG, "data");
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(), true, config);
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(
+ StandardCharsets.UTF_8), true, config);
Assert.assertFalse(m.getMetadata().containsKey("data"));
checkMergedData(m);
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
checkEnvelopeMetadata(m);
checkKafkaMetadata(m, true);
checkAppMetadata(m, false);
}
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(), false, config);
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(
+ StandardCharsets.UTF_8), false, config);
Assert.assertFalse(m.getMetadata().containsKey("data"));
checkMergedData(m);
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
Assert.assertTrue(m.getMetadata().isEmpty());
}
}
@@ -265,19 +281,21 @@ public class RawMessageUtilTest {
Tuple t = createTuple(new HashMap<>() , "{}");
Map<String, Object> config = ImmutableMap.of(EnvelopedRawMessageStrategy.MESSAGE_FIELD_CONFIG, "data");
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(), true, config);
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(
+ StandardCharsets.UTF_8), true, config);
Assert.assertFalse(m.getMetadata().containsKey("data"));
checkMergedData(m);
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
checkEnvelopeMetadata(m);
checkKafkaMetadata(m, true);
checkAppMetadata(m, true);
}
{
- RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(), false, config);
+ RawMessage m = RawMessageUtil.INSTANCE.getRawMessage( RawMessageStrategies.ENVELOPE, t, envelopedData.getBytes(
+ StandardCharsets.UTF_8), false, config);
Assert.assertFalse(m.getMetadata().containsKey("data"));
checkMergedData(m);
- Assert.assertEquals("raw_message", new String(m.getMessage()));
+ Assert.assertEquals("raw_message", new String(m.getMessage(), StandardCharsets.UTF_8));
Assert.assertTrue(m.getMetadata().isEmpty());
}
}
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
index 910fe27..7c1ff8c 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
@@ -30,6 +30,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.HashMap;
import java.util.List;
@@ -113,7 +114,7 @@ public class ConfigurationsUtils {
* @throws Exception If there's an error writing to ZK
*/
public static void writeGlobalConfigToZookeeper(byte[] globalConfig, CuratorFramework client) throws Exception {
- GLOBAL.deserialize(new String(globalConfig));
+ GLOBAL.deserialize(new String(globalConfig, StandardCharsets.UTF_8));
writeToZookeeper(GLOBAL.getZookeeperRoot(), globalConfig, client);
}
@@ -125,7 +126,7 @@ public class ConfigurationsUtils {
* @throws Exception If there's an error writing to ZK
*/
public static void writeProfilerConfigToZookeeper(byte[] config, CuratorFramework client) throws Exception {
- PROFILER.deserialize(new String(config));
+ PROFILER.deserialize(new String(config, StandardCharsets.UTF_8));
writeToZookeeper(PROFILER.getZookeeperRoot(), config, client);
}
@@ -165,7 +166,8 @@ public class ConfigurationsUtils {
* @throws Exception If there's an error writing to ZK
*/
public static void writeSensorParserConfigToZookeeper(String sensorType, byte[] configData, CuratorFramework client) throws Exception {
- SensorParserConfig c = (SensorParserConfig) PARSER.deserialize(new String(configData));
+ SensorParserConfig c = (SensorParserConfig) PARSER.deserialize(new String(configData,
+ StandardCharsets.UTF_8));
c.init();
writeToZookeeper(PARSER.getZookeeperRoot() + "/" + sensorType, configData, client);
}
@@ -206,7 +208,7 @@ public class ConfigurationsUtils {
* @throws Exception If there's an error writing to ZK
*/
public static void writeSensorIndexingConfigToZookeeper(String sensorType, byte[] configData, CuratorFramework client) throws Exception {
- INDEXING.deserialize(new String(configData));
+ INDEXING.deserialize(new String(configData, StandardCharsets.UTF_8));
writeToZookeeper(INDEXING.getZookeeperRoot() + "/" + sensorType, configData, client);
}
@@ -246,7 +248,7 @@ public class ConfigurationsUtils {
* @throws Exception If there's an error writing to ZK
*/
public static void writeSensorEnrichmentConfigToZookeeper(String sensorType, byte[] configData, CuratorFramework client) throws Exception {
- ENRICHMENT.deserialize(new String(configData));
+ ENRICHMENT.deserialize(new String(configData, StandardCharsets.UTF_8));
writeToZookeeper(ENRICHMENT.getZookeeperRoot() + "/" + sensorType, configData, client);
}
@@ -644,7 +646,8 @@ public class ConfigurationsUtils {
case GLOBAL:
final byte[] globalConfig = readGlobalConfigFromFile(rootFilePath);
if (globalConfig.length > 0) {
- setupStellarStatically(client, Optional.of(new String(globalConfig)));
+ setupStellarStatically(client, Optional.of(new String(globalConfig,
+ StandardCharsets.UTF_8)));
writeGlobalConfigToZookeeper(globalConfig, client);
}
break;
@@ -696,7 +699,7 @@ public class ConfigurationsUtils {
if (globalConfigPath != null) {
final byte[] globalConfig = readGlobalConfigFromFile(globalConfigPath);
if (globalConfig.length > 0) {
- setupStellarStatically(client, Optional.of(new String(globalConfig)));
+ setupStellarStatically(client, Optional.of(new String(globalConfig, StandardCharsets.UTF_8)));
ConfigurationsUtils.writeGlobalConfigToZookeeper(readGlobalConfigFromFile(globalConfigPath), client);
}
}
@@ -752,7 +755,7 @@ public class ConfigurationsUtils {
setupStellarStatically(client, Optional.empty());
}
else {
- setupStellarStatically(client, Optional.of(new String(ret)));
+ setupStellarStatically(client, Optional.of(new String(ret, StandardCharsets.UTF_8)));
}
}
@@ -949,7 +952,7 @@ public class ConfigurationsUtils {
byte[] prettyPatchedConfig = JSONUtils.INSTANCE.applyPatch(patchData, configData);
// ensure the patch produces a valid result; otherwise exception thrown during deserialization
- String prettyPatchedConfigStr = new String(prettyPatchedConfig);
+ String prettyPatchedConfigStr = new String(prettyPatchedConfig, StandardCharsets.UTF_8);
configurationType.deserialize(prettyPatchedConfigStr);
writeConfigToZookeeper(configurationType, configName, prettyPatchedConfig, client);
@@ -995,21 +998,22 @@ public class ConfigurationsUtils {
if (configType.equals(GLOBAL)) {
byte[] globalConfigData = client.getData().forPath(configType.getZookeeperRoot());
- callback.visit(configType, "global", new String(globalConfigData));
+ callback.visit(configType, "global", new String(globalConfigData, StandardCharsets.UTF_8));
}
else if(configType.equals(PROFILER)) {
byte[] profilerConfigData = client.getData().forPath(configType.getZookeeperRoot());
- callback.visit(configType, "profiler", new String(profilerConfigData));
+ callback.visit(configType, "profiler", new String(profilerConfigData,
+ StandardCharsets.UTF_8));
}
else if (configType.equals(PARSER) || configType.equals(ENRICHMENT) || configType.equals(INDEXING)) {
if (configName.isPresent()) {
byte[] data = readConfigBytesFromZookeeper(configType, configName, client);
- callback.visit(configType, configName.get(), new String(data));
+ callback.visit(configType, configName.get(), new String(data, StandardCharsets.UTF_8));
} else {
List<String> children = client.getChildren().forPath(configType.getZookeeperRoot());
for (String child : children) {
byte[] data = client.getData().forPath(configType.getZookeeperRoot() + "/" + child);
- callback.visit(configType, child, new String(data));
+ callback.visit(configType, child, new String(data, StandardCharsets.UTF_8));
}
}
}
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
index 9797c22..2ed4b87 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
@@ -18,6 +18,7 @@
package org.apache.metron.common.configuration;
import com.fasterxml.jackson.core.JsonProcessingException;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.lang3.builder.ToStringBuilder;
@@ -432,7 +433,7 @@ public class SensorParserConfig implements Serializable {
* @throws IOException If the config cannot be loaded
*/
public static SensorParserConfig fromBytes(byte[] config) throws IOException {
- SensorParserConfig ret = JSONUtils.INSTANCE.load(new String(config), SensorParserConfig.class);
+ SensorParserConfig ret = JSONUtils.INSTANCE.load(new String(config, StandardCharsets.UTF_8), SensorParserConfig.class);
ret.init();
return ret;
}
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java
index 0bfc7a8..1338ba2 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentConfig.java
@@ -18,6 +18,7 @@
package org.apache.metron.common.configuration.enrichment;
import com.fasterxml.jackson.core.JsonProcessingException;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.common.configuration.enrichment.threatintel.ThreatIntelConfig;
import org.apache.metron.common.utils.JSONUtils;
@@ -94,7 +95,7 @@ public class SensorEnrichmentConfig {
}
public static SensorEnrichmentConfig fromBytes(byte[] config) throws IOException {
- return JSONUtils.INSTANCE.load(new String(config), SensorEnrichmentConfig.class);
+ return JSONUtils.INSTANCE.load(new String(config, StandardCharsets.UTF_8), SensorEnrichmentConfig.class);
}
public String toJSON() throws JsonProcessingException {
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentUpdateConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentUpdateConfig.java
index 08bd629..0b5888c 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentUpdateConfig.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentUpdateConfig.java
@@ -20,6 +20,7 @@ package org.apache.metron.common.configuration.enrichment;
import com.google.common.base.Joiner;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -114,7 +115,8 @@ public class SensorEnrichmentUpdateConfig {
@Override
public void persistConfig(String sensor, SensorEnrichmentConfig config) throws Exception {
- ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensor, config.toJSON().getBytes(), client);
+ ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensor, config.toJSON().getBytes(
+ StandardCharsets.UTF_8), client);
}
}
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
index c4df1b4..1313eb2 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
@@ -19,6 +19,7 @@ package org.apache.metron.common.configuration.profiler;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.lang.builder.ToStringBuilder;
@@ -287,7 +288,7 @@ public class ProfileConfig implements Serializable {
* @throws IOException If unable to deserialize the bytes into a {@link ProfileConfig}
*/
public static ProfileConfig fromBytes(byte[] bytes) throws IOException {
- return JSONUtils.INSTANCE.load(new String(bytes), ProfileConfig.class);
+ return JSONUtils.INSTANCE.load(new String(bytes, StandardCharsets.UTF_8), ProfileConfig.class);
}
/**
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java
index f5c4981..33cd1f7 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfig.java
@@ -20,6 +20,7 @@ package org.apache.metron.common.configuration.profiler;
import com.fasterxml.jackson.annotation.JsonGetter;
import com.fasterxml.jackson.annotation.JsonSetter;
import com.fasterxml.jackson.core.JsonProcessingException;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.lang3.builder.ToStringBuilder;
@@ -135,7 +136,7 @@ public class ProfilerConfig implements Serializable {
* @throws IOException If there's an error deserializing the raw bytes
*/
public static ProfilerConfig fromBytes(byte[] bytes) throws IOException {
- return JSONUtils.INSTANCE.load(new String(bytes), ProfilerConfig.class);
+ return JSONUtils.INSTANCE.load(new String(bytes, StandardCharsets.UTF_8), ProfilerConfig.class);
}
/**
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java
index fe6a161..d074fde 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java
@@ -184,7 +184,7 @@ public class MetronError {
// It's unclear if we need a rawMessageBytes field so commenting out for now
//String rawMessageBytesField = rawMessages.size() == 1 ? ErrorFields.RAW_MESSAGE_BYTES.getName() : ErrorFields.RAW_MESSAGE_BYTES.getName() + "_" + i;
if(rawMessage instanceof byte[]) {
- errorMessage.put(rawMessageField, new String((byte[])rawMessage, Charset.forName("UTF-8")));
+ errorMessage.put(rawMessageField, new String((byte[])rawMessage, UTF_8));
//errorMessage.put(rawMessageBytesField, com.google.common.primitives.Bytes.asList((byte[])rawMessage));
} else if (rawMessage instanceof JSONObject) {
JSONObject rawMessageJSON = (JSONObject) rawMessage;
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/metadata/EnvelopedRawMessageStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/metadata/EnvelopedRawMessageStrategy.java
index d485245..9b1d501 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/metadata/EnvelopedRawMessageStrategy.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/message/metadata/EnvelopedRawMessageStrategy.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.common.message.metadata;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
import org.json.simple.JSONObject;
@@ -70,7 +71,8 @@ public class EnvelopedRawMessageStrategy implements RawMessageStrategy {
try {
String prefix = MetadataUtil.INSTANCE.getMetadataPrefix(config);
- Map<String, Object> extraMetadata = JSONUtils.INSTANCE.load(new String(envelope), JSONUtils.MAP_SUPPLIER);
+ Map<String, Object> extraMetadata = JSONUtils.INSTANCE.load(new String(envelope,
+ StandardCharsets.UTF_8), JSONUtils.MAP_SUPPLIER);
String message = null;
if(extraMetadata != null) {
for(Map.Entry<String, Object> kv : extraMetadata.entrySet()) {
@@ -83,13 +85,13 @@ public class EnvelopedRawMessageStrategy implements RawMessageStrategy {
if(message != null) {
if(!readMetadata) {
LOG.debug("Ignoring metadata; Message: " + message + " rawMetadata: " + rawMetadata + " and field = " + messageField);
- return new RawMessage(message.getBytes(), new HashMap<>());
+ return new RawMessage(message.getBytes(StandardCharsets.UTF_8), new HashMap<>());
}
else {
//remove the message field from the metadata since it's data, not metadata.
rawMetadata.remove(MetadataUtil.INSTANCE.prefixKey(prefix, messageField));
LOG.debug("Attaching metadata; Message: " + message + " rawMetadata: " + rawMetadata + " and field = " + messageField);
- return new RawMessage(message.getBytes(), rawMetadata);
+ return new RawMessage(message.getBytes(StandardCharsets.UTF_8), rawMetadata);
}
}
} catch (IOException e) {
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
index 5cd25a2..9689214 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
@@ -21,6 +21,7 @@ package org.apache.metron.common.utils;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
+import java.nio.charset.StandardCharsets;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
@@ -64,7 +65,7 @@ public enum KafkaUtils {
List<String> ret = new ArrayList<>();
for(String id : client.getChildren().forPath("/brokers/ids")) {
byte[] data = client.getData().forPath("/brokers/ids/" + id);
- String brokerInfoStr = new String(data);
+ String brokerInfoStr = new String(data, StandardCharsets.UTF_8);
Map<String, Object> brokerInfo = JSONUtils.INSTANCE.load(brokerInfoStr, JSONUtils.MAP_SUPPLIER);
String host = (String) brokerInfo.get("host");
if(host != null) {
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/ConfigurationsUpdater.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/ConfigurationsUpdater.java
index 53aedd7..da0c6ed 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/ConfigurationsUpdater.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/zookeeper/configurations/ConfigurationsUpdater.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.common.zookeeper.configurations;
+import java.nio.charset.StandardCharsets;
import org.apache.curator.framework.CuratorFramework;
import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.Configurations;
@@ -65,11 +66,13 @@ public abstract class ConfigurationsUpdater<T extends Configurations> implements
if (data.length != 0) {
String name = path.substring(path.lastIndexOf("/") + 1);
if (path.startsWith(getType().getZookeeperRoot())) {
- LOG.debug("Updating the {} config: {} -> {}", () -> getType().name(), () -> name, () -> new String(data == null?"".getBytes():data));
+ LOG.debug("Updating the {} config: {} -> {}", () -> getType().name(), () -> name,
+ () -> new String(data == null?"".getBytes(StandardCharsets.UTF_8):data, StandardCharsets.UTF_8));
update(name, data);
reloadCallback(name, getType());
} else if (ConfigurationType.GLOBAL.getZookeeperRoot().equals(path)) {
- LOG.debug("Updating the global config: {}", () -> new String(data == null?"".getBytes():data));
+ LOG.debug("Updating the global config: {}",
+ () -> new String(data == null?"".getBytes(StandardCharsets.UTF_8):data, StandardCharsets.UTF_8));
getConfigurations().updateGlobalConfig(data);
reloadCallback(name, ConfigurationType.GLOBAL);
}
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java
index 438da3c..3c9470b 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java
@@ -36,6 +36,7 @@ import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryNotEmptyException;
import java.nio.file.Files;
import java.nio.file.Paths;
@@ -144,11 +145,13 @@ public class ConfigurationManagerIntegrationTest {
private void validateConfigsOnDisk(File configDir) throws IOException {
File globalConfigFile = new File(configDir, "global.json");
Assert.assertTrue("Global config does not exist", globalConfigFile.exists());
- validateConfig("global", GLOBAL, new String(Files.readAllBytes(Paths.get(globalConfigFile.toURI()))));
+ validateConfig("global", GLOBAL, new String(Files.readAllBytes(Paths.get(globalConfigFile.toURI())),
+ StandardCharsets.UTF_8));
for(String sensor : sensors) {
File sensorFile = new File(configDir, ENRICHMENT.getDirectory() + "/" + sensor + ".json");
Assert.assertTrue(sensor + " config does not exist", sensorFile.exists());
- validateConfig(sensor, ENRICHMENT, new String(Files.readAllBytes(Paths.get(sensorFile.toURI()))));
+ validateConfig(sensor, ENRICHMENT, new String(Files.readAllBytes(Paths.get(sensorFile.toURI())),
+ StandardCharsets.UTF_8));
}
}
@@ -305,7 +308,7 @@ public class ConfigurationManagerIntegrationTest {
private String redirectSystemOut(final String[] args, RedirectCallback callback) throws Exception {
PrintStream os = System.out;
- try (OutputStream baos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(baos)) {
+ try (OutputStream baos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(baos, false, StandardCharsets.UTF_8.name())) {
System.setOut(ps);
callback.call(args);
System.out.flush();
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationTest.java
index 3031ab9..04d7e79 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.common.configuration;
+import java.nio.charset.StandardCharsets;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.ExistsBuilder;
import org.apache.curator.framework.api.GetChildrenBuilder;
@@ -73,7 +74,7 @@ public class ConfigurationTest {
JSONObject global = new JSONObject();
global.put(TEST_PROPERTY, TEST_VALUE);
- return global.toString().getBytes();
+ return global.toString().getBytes(StandardCharsets.UTF_8);
}
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ParserConfigurationsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ParserConfigurationsTest.java
index c099aec..77e9699 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ParserConfigurationsTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ParserConfigurationsTest.java
@@ -24,6 +24,7 @@ import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertThat;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import org.adrianwalker.multilinestring.Multiline;
@@ -77,7 +78,8 @@ public class ParserConfigurationsTest {
@Test
public void sensorParserConfig_properties_populated_by_JSON_configuration() throws IOException {
ParserConfigurations parserConfigs = new ParserConfigurations();
- parserConfigs.updateSensorParserConfig("test-sensor", parserConfig.getBytes());
+ parserConfigs.updateSensorParserConfig("test-sensor", parserConfig.getBytes(
+ StandardCharsets.UTF_8));
SensorParserConfig actualSensorConfig = parserConfigs.getSensorParserConfig("test-sensor");
assertThat(actualSensorConfig.getParserClassName(), equalTo("parser-class"));
assertThat(actualSensorConfig.getFilterClassName(), equalTo("filter-class"));
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java
index 5a77b4f..a17f8d6 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java
@@ -17,6 +17,9 @@
*/
package org.apache.metron.common.configuration;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import nl.jqno.equalsverifier.EqualsVerifier;
import nl.jqno.equalsverifier.Warning;
import org.apache.commons.io.IOUtils;
@@ -47,11 +50,12 @@ public class SensorEnrichmentConfigTest {
public void testSerDe() throws IOException {
for(File enrichmentConfig : new File(new File(TestConstants.ENRICHMENTS_CONFIGS_PATH), "enrichments").listFiles()) {
SensorEnrichmentConfig config = null;
- try (BufferedReader br = new BufferedReader(new FileReader(enrichmentConfig))) {
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(enrichmentConfig), StandardCharsets.UTF_8))) {
String parserStr = IOUtils.toString(br);
- config = SensorEnrichmentConfig.fromBytes(parserStr.getBytes());
+ config = SensorEnrichmentConfig.fromBytes(parserStr.getBytes(StandardCharsets.UTF_8));
}
- SensorEnrichmentConfig config2 = SensorEnrichmentConfig.fromBytes(config.toJSON().getBytes());
+ SensorEnrichmentConfig config2 = SensorEnrichmentConfig.fromBytes(config.toJSON().getBytes(
+ StandardCharsets.UTF_8));
Assert.assertEquals(config2, config);
}
}
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorParserConfigTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorParserConfigTest.java
index fe69ed9..ea477fc 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorParserConfigTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorParserConfigTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.common.configuration;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.io.IOUtils;
import org.apache.metron.TestConstants;
import org.junit.Assert;
@@ -30,11 +31,12 @@ public class SensorParserConfigTest {
public void testSerDe() throws IOException {
for(File parserConfig : new File(new File(TestConstants.PARSER_CONFIGS_PATH), "parsers").listFiles()) {
SensorParserConfig config = null;
- try (BufferedReader br = new BufferedReader(new FileReader(parserConfig))) {
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(parserConfig), StandardCharsets.UTF_8))) {
String parserStr = IOUtils.toString(br);
- config = SensorParserConfig.fromBytes(parserStr.getBytes());
+ config = SensorParserConfig.fromBytes(parserStr.getBytes(StandardCharsets.UTF_8));
}
- SensorParserConfig config2 = SensorParserConfig.fromBytes(config.toJSON().getBytes());
+ SensorParserConfig config2 = SensorParserConfig.fromBytes(config.toJSON().getBytes(
+ StandardCharsets.UTF_8));
Assert.assertEquals(config2, config);
}
}
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/writer/EnrichmentWriterConfigurationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/writer/EnrichmentWriterConfigurationTest.java
index b6aae19..8e9e17b 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/writer/EnrichmentWriterConfigurationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/writer/EnrichmentWriterConfigurationTest.java
@@ -23,6 +23,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.EnrichmentConfigurations;
import org.junit.Test;
@@ -41,7 +42,7 @@ public class EnrichmentWriterConfigurationTest {
@Test
public void gets_batch_size_and_timeout_from_global_config() throws IOException {
EnrichmentConfigurations configs = new EnrichmentConfigurations();
- configs.updateGlobalConfig(globalJson.getBytes());
+ configs.updateGlobalConfig(globalJson.getBytes(StandardCharsets.UTF_8));
EnrichmentWriterConfiguration writerConfig = new EnrichmentWriterConfiguration(configs);
assertThat("batch timeout should match global config setting",
writerConfig.getBatchTimeout(null), equalTo(555));
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/writer/ParserWriterConfigurationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/writer/ParserWriterConfigurationTest.java
index ebe3d9b..47fde10 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/writer/ParserWriterConfigurationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/writer/ParserWriterConfigurationTest.java
@@ -22,6 +22,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.ParserConfigurations;
import org.junit.Assert;
@@ -60,7 +61,8 @@ public class ParserWriterConfigurationTest {
public void pulls_writer_configuration_from_parserConfig() throws IOException {
ParserConfigurations parserConfigurations = new ParserConfigurations();
final String sensorName = "some-sensor";
- parserConfigurations.updateSensorParserConfig("some-sensor", configJson.getBytes());
+ parserConfigurations.updateSensorParserConfig("some-sensor", configJson.getBytes(
+ StandardCharsets.UTF_8));
ParserWriterConfiguration writerConfiguration = new ParserWriterConfiguration(
parserConfigurations);
assertThat("batch size should match", writerConfiguration.getBatchSize(sensorName), equalTo(5));
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/writer/ProfilerWriterConfigurationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/writer/ProfilerWriterConfigurationTest.java
index 3639276..244cc6f 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/writer/ProfilerWriterConfigurationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/writer/ProfilerWriterConfigurationTest.java
@@ -23,6 +23,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.profiler.ProfilerConfigurations;
import org.junit.Test;
@@ -41,7 +42,7 @@ public class ProfilerWriterConfigurationTest {
@Test
public void gets_batch_size_and_timeout_from_global_config() throws IOException {
ProfilerConfigurations configs = new ProfilerConfigurations();
- configs.updateGlobalConfig(globalJson.getBytes());
+ configs.updateGlobalConfig(globalJson.getBytes(StandardCharsets.UTF_8));
ProfilerWriterConfiguration writerConfig = new ProfilerWriterConfiguration(configs);
assertThat("batch timeout should match global config setting",
writerConfig.getBatchTimeout(null), equalTo(555));
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java
index 294d6dc..2203ff5 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
import com.google.common.collect.Sets;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -97,7 +98,7 @@ public class MetronErrorTest {
assertEquals("{\"value\":\"message1\"}", errorJSON.get(Constants.ErrorFields.RAW_MESSAGE.getName() + "_0"));
assertEquals("{\"value\":\"message2\"}", errorJSON.get(Constants.ErrorFields.RAW_MESSAGE.getName() + "_1"));
- error = new MetronError().addRawMessage("raw message".getBytes());
+ error = new MetronError().addRawMessage("raw message".getBytes(StandardCharsets.UTF_8));
errorJSON = error.getJSONObject();
assertEquals("raw message", errorJSON.get(Constants.ErrorFields.RAW_MESSAGE.getName()));
// It's unclear if we need a rawMessageBytes field so commenting out for now
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/FieldNameConvertersTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/FieldNameConvertersTest.java
index 2c263f2..baf2932 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/FieldNameConvertersTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/FieldNameConvertersTest.java
@@ -18,6 +18,7 @@
package org.apache.metron.common.field;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.IndexingConfigurations;
import org.apache.metron.common.configuration.writer.IndexingWriterConfiguration;
@@ -34,7 +35,7 @@ public class FieldNameConvertersTest {
private WriterConfiguration createConfig(String writer, String sensor, String json) throws Exception {
IndexingConfigurations indexingConfig = new IndexingConfigurations();
- indexingConfig.updateSensorIndexingConfig(sensor, json.getBytes());
+ indexingConfig.updateSensorIndexingConfig(sensor, json.getBytes(StandardCharsets.UTF_8));
return new IndexingWriterConfiguration(writer, indexingConfig);
}
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/typosquat/TyposquattingStrategiesTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/typosquat/TyposquattingStrategiesTest.java
index d7f99f0..8cea42a 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/typosquat/TyposquattingStrategiesTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/typosquat/TyposquattingStrategiesTest.java
@@ -22,6 +22,8 @@ import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
+import java.io.FileInputStream;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.stellar.common.utils.StellarProcessorUtils;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -56,7 +58,7 @@ public class TyposquattingStrategiesTest {
@BeforeClass
public static void setup() throws Exception {
for(Map.Entry<String, EnumMap<TyposquattingStrategies, Set<String>>> kv : expected.entrySet()) {
- try(BufferedReader br = new BufferedReader(new FileReader( "src/test/resources/typosquat/" + kv.getKey() + ".csv") ) )
+ try(BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream("src/test/resources/typosquat/" + kv.getKey() + ".csv"), StandardCharsets.UTF_8) ) )
{
for(String line = null;(line = br.readLine()) != null;) {
if(line.startsWith("#")) {
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
index 4676591..fed333b 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertThat;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.File;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import org.adrianwalker.multilinestring.Multiline;
@@ -121,7 +122,8 @@ public class JSONUtilsTest {
@Test
public void applyPatch_modifies_source_json_doc() throws IOException {
- String actual = new String(JSONUtils.INSTANCE.applyPatch(patchJson, sourceJson));
+ String actual = new String(JSONUtils.INSTANCE.applyPatch(patchJson, sourceJson),
+ StandardCharsets.UTF_8);
assertThat(JSONUtils.INSTANCE.load(actual, JSONUtils.MAP_SUPPLIER), equalTo(JSONUtils.INSTANCE.load(expectedJson, JSONUtils.MAP_SUPPLIER)));
}
@@ -157,7 +159,8 @@ public class JSONUtilsTest {
@Test
public void applyPatch_modifies_complex_source_json_doc() throws IOException {
- String actual = new String(JSONUtils.INSTANCE.applyPatch(patchComplexJson, complexJson));
+ String actual = new String(JSONUtils.INSTANCE.applyPatch(patchComplexJson, complexJson),
+ StandardCharsets.UTF_8);
assertThat(JSONUtils.INSTANCE.load(actual, JSONUtils.MAP_SUPPLIER), equalTo(JSONUtils.INSTANCE.load(expectedComplexJson, JSONUtils.MAP_SUPPLIER)));
}
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/KafkaUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/KafkaUtilsTest.java
index 72ac51e..2c4a5d5 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/KafkaUtilsTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/KafkaUtilsTest.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.when;
import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -65,7 +66,8 @@ public class KafkaUtilsTest {
when(client.getChildren()).thenReturn(childrenBuilder);
when(childrenBuilder.forPath("/brokers/ids")).thenReturn(brokerIds);
when(client.getData()).thenReturn(dataBuilder);
- when(dataBuilder.forPath("/brokers/ids/1")).thenReturn(brokerWithHostPort.getBytes());
+ when(dataBuilder.forPath("/brokers/ids/1")).thenReturn(brokerWithHostPort.getBytes(
+ StandardCharsets.UTF_8));
ArrayList<String> expected = new ArrayList<>();
expected.add("192.168.1.148:9092");
@@ -88,7 +90,8 @@ public class KafkaUtilsTest {
when(client.getChildren()).thenReturn(childrenBuilder);
when(childrenBuilder.forPath("/brokers/ids")).thenReturn(brokerIds);
when(client.getData()).thenReturn(dataBuilder);
- when(dataBuilder.forPath("/brokers/ids/1")).thenReturn(brokerWithEndpoints.getBytes());
+ when(dataBuilder.forPath("/brokers/ids/1")).thenReturn(brokerWithEndpoints.getBytes(
+ StandardCharsets.UTF_8));
ArrayList<String> expected = new ArrayList<>();
expected.add("host1:9092");
@@ -117,7 +120,7 @@ public class KafkaUtilsTest {
when(childrenBuilder.forPath("/brokers/ids")).thenReturn(brokerIds);
when(client.getData()).thenReturn(dataBuilder);
when(dataBuilder.forPath("/brokers/ids/1"))
- .thenReturn(brokerWithHostPortAndEndpoints.getBytes());
+ .thenReturn(brokerWithHostPortAndEndpoints.getBytes(StandardCharsets.UTF_8));
ArrayList<String> expected = new ArrayList<>();
expected.add("192.168.1.148:9092");
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/file/ReaderSpliteratorTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/file/ReaderSpliteratorTest.java
index eab0ee5..48ab168 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/file/ReaderSpliteratorTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/file/ReaderSpliteratorTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.common.utils.file;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
@@ -67,13 +68,13 @@ public class ReaderSpliteratorTest {
if(dataFile.exists()) {
dataFile.delete();
}
- Files.write(dataFile.toPath(), data.getBytes(), StandardOpenOption.CREATE_NEW, StandardOpenOption.TRUNCATE_EXISTING);
+ Files.write(dataFile.toPath(), data.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE_NEW, StandardOpenOption.TRUNCATE_EXISTING);
dataFile.deleteOnExit();
}
public static BufferedReader getReader() {
try {
- return new BufferedReader(new FileReader(dataFile));
+ return new BufferedReader(new InputStreamReader(new FileInputStream(dataFile), StandardCharsets.UTF_8));
} catch (FileNotFoundException e) {
throw new IllegalStateException(e.getMessage(), e);
}
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java
index ce898d3..9deaab0 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.common.zookeeper;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.io.IOUtils;
import org.apache.curator.framework.CuratorFramework;
@@ -217,11 +218,15 @@ public class ZKConfigurationsCacheIntegrationTest {
@Test
public void validateUpdate() throws Exception {
- ConfigurationsUtils.writeSensorIndexingConfigToZookeeper("test", testIndexingConfig.getBytes(), client);
- ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig.getBytes(), client);
- ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper("test", testEnrichmentConfig.getBytes(), client);
- ConfigurationsUtils.writeSensorParserConfigToZookeeper("bro", testParserConfig.getBytes(), client);
- ConfigurationsUtils.writeProfilerConfigToZookeeper( profilerConfig.getBytes(), client);
+ ConfigurationsUtils.writeSensorIndexingConfigToZookeeper("test", testIndexingConfig.getBytes(
+ StandardCharsets.UTF_8), client);
+ ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig.getBytes(StandardCharsets.UTF_8), client);
+ ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper("test", testEnrichmentConfig.getBytes(
+ StandardCharsets.UTF_8), client);
+ ConfigurationsUtils.writeSensorParserConfigToZookeeper("bro", testParserConfig.getBytes(
+ StandardCharsets.UTF_8), client);
+ ConfigurationsUtils.writeProfilerConfigToZookeeper( profilerConfig.getBytes(
+ StandardCharsets.UTF_8), client);
//indexing
{
Map<String, Object> expectedConfig = JSONUtils.INSTANCE.load(testIndexingConfig, JSONUtils.MAP_SUPPLIER);
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/integration/components/ConfigUploadComponent.java b/metron-platform/metron-common/src/test/java/org/apache/metron/integration/components/ConfigUploadComponent.java
index c9f9f70..47c31e2 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/integration/components/ConfigUploadComponent.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/integration/components/ConfigUploadComponent.java
@@ -17,21 +17,19 @@
*/
package org.apache.metron.integration.components;
-import static org.apache.metron.common.configuration.ConfigurationsUtils.getClient;
-import static org.apache.metron.common.configuration.ConfigurationsUtils.readSensorParserConfigFromZookeeper;
-import static org.apache.metron.common.configuration.ConfigurationsUtils.uploadConfigsToZookeeper;
-import static org.apache.metron.common.configuration.ConfigurationsUtils.writeGlobalConfigToZookeeper;
-import static org.apache.metron.common.configuration.ConfigurationsUtils.writeSensorParserConfigToZookeeper;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.integration.InMemoryComponent;
+import org.apache.metron.integration.UnableToStartException;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.function.Consumer;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.metron.common.configuration.SensorParserConfig;
-import org.apache.metron.integration.InMemoryComponent;
-import org.apache.metron.integration.UnableToStartException;
+
+import static org.apache.metron.common.configuration.ConfigurationsUtils.*;
public class ConfigUploadComponent implements InMemoryComponent {
@@ -153,7 +151,7 @@ public class ConfigUploadComponent implements InMemoryComponent {
}
if(globalConfig.isPresent()) {
- writeGlobalConfigToZookeeper(globalConfig.get().getBytes(), zookeeperUrl);
+ writeGlobalConfigToZookeeper(globalConfig.get().getBytes(StandardCharsets.UTF_8), zookeeperUrl);
}
if(postStartCallback.isPresent()) {
postStartCallback.get().accept(this);
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/location/RawLocation.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/location/RawLocation.java
index 5f2db33..02f841f 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/location/RawLocation.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/location/RawLocation.java
@@ -18,6 +18,7 @@
package org.apache.metron.dataloads.nonbulk.flatfile.location;
import java.io.*;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Optional;
import java.util.zip.GZIPInputStream;
@@ -38,20 +39,22 @@ public interface RawLocation<T> {
default BufferedReader openReader(String loc) throws IOException {
InputStream is = openInputStream(loc);
if(loc.endsWith(".gz")) {
- return new BufferedReader(new InputStreamReader(new GZIPInputStream(is)));
+ return new BufferedReader(new InputStreamReader(new GZIPInputStream(is),
+ StandardCharsets.UTF_8));
}
else if(loc.endsWith(".zip")) {
ZipInputStream zis = new ZipInputStream(is);
ZipEntry entry = zis.getNextEntry();
if(entry != null) {
- return new BufferedReader(new InputStreamReader(zis));
+ return new BufferedReader(new InputStreamReader(zis, StandardCharsets.UTF_8));
}
else {
- return new BufferedReader(new InputStreamReader(new ByteArrayInputStream(new byte[] {})));
+ return new BufferedReader(new InputStreamReader(new ByteArrayInputStream(new byte[] {}),
+ StandardCharsets.UTF_8));
}
}
else {
- return new BufferedReader(new InputStreamReader(is));
+ return new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
}
}
}
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java
index dc078ba..68855a6 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java
@@ -19,6 +19,7 @@ package org.apache.metron.dataloads.extractor.stix;
import com.google.common.base.Joiner;
import com.google.common.collect.Iterables;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
@@ -39,8 +40,9 @@ public class StixExtractorTest {
@Before
public void setup() throws IOException {
- stixDoc = Joiner.on("\n").join(IOUtils.readLines(new FileReader(new File("src/test/resources/stix_example.xml"))));
- stixDocWithoutCondition = Joiner.on("\n").join(IOUtils.readLines(new FileReader(new File("src/test/resources/stix_example_wo_conditions.xml"))));
+ stixDoc = Joiner.on("\n").join(IOUtils.readLines(new InputStreamReader(new FileInputStream(new File("src/test/resources/stix_example.xml")),
+ StandardCharsets.UTF_8)));
+ stixDocWithoutCondition = Joiner.on("\n").join(IOUtils.readLines(new InputStreamReader(new FileInputStream(new File("src/test/resources/stix_example_wo_conditions.xml")), StandardCharsets.UTF_8)));
}
/**
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/HBaseUtil.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/HBaseUtil.java
index 48a9ba3..93b3eac 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/HBaseUtil.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/HBaseUtil.java
@@ -18,16 +18,6 @@
package org.apache.metron.dataloads.hbase.mr;
import com.google.common.base.Joiner;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.util.AbstractMap;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -36,6 +26,14 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
/**
* Spin up a test HBase cluster
*/
@@ -68,7 +66,7 @@ public enum HBaseUtil {
public void writeFile(String contents, Path filename, FileSystem fs) throws IOException {
FSDataOutputStream os = fs.create(filename, true);
- PrintWriter pw = new PrintWriter(new OutputStreamWriter(os));
+ PrintWriter pw = new PrintWriter(new OutputStreamWriter(os, StandardCharsets.UTF_8));
pw.print(contents);
pw.flush();
os.close();
@@ -76,7 +74,7 @@ public enum HBaseUtil {
public String readFile(FileSystem fs, Path filename) throws IOException {
FSDataInputStream in = fs.open(filename);
- BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
List<String> contents = new ArrayList<>();
for(String line = null;(line = br.readLine()) != null;) {
contents.add(line);
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoaderIntegrationTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoaderIntegrationTest.java
index 443d39d..a1d98f6 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoaderIntegrationTest.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoaderIntegrationTest.java
@@ -17,6 +17,8 @@
*/
package org.apache.metron.dataloads.nonbulk.flatfile;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.PosixParser;
@@ -203,42 +205,44 @@ public class SimpleEnrichmentFlatFileLoaderIntegrationTest {
lineByLineExtractorConfigFile.delete();
}
Files.write( lineByLineExtractorConfigFile.toPath()
- , lineByLineExtractorConfig.getBytes()
+ , lineByLineExtractorConfig.getBytes(StandardCharsets.UTF_8)
, StandardOpenOption.CREATE_NEW , StandardOpenOption.TRUNCATE_EXISTING
);
if(wholeFileExtractorConfigFile.exists()) {
wholeFileExtractorConfigFile.delete();
}
Files.write( wholeFileExtractorConfigFile.toPath()
- , wholeFileExtractorConfig.getBytes()
+ , wholeFileExtractorConfig.getBytes(StandardCharsets.UTF_8)
, StandardOpenOption.CREATE_NEW , StandardOpenOption.TRUNCATE_EXISTING
);
if(stellarExtractorConfigFile.exists()) {
stellarExtractorConfigFile.delete();
}
Files.write( stellarExtractorConfigFile.toPath()
- , stellarExtractorConfig.replace("%ZK_QUORUM%", zookeeperUrl).getBytes()
+ , stellarExtractorConfig.replace("%ZK_QUORUM%", zookeeperUrl).getBytes(
+ StandardCharsets.UTF_8)
, StandardOpenOption.CREATE_NEW , StandardOpenOption.TRUNCATE_EXISTING
);
if(customLineByLineExtractorConfigFile.exists()) {
customLineByLineExtractorConfigFile.delete();
}
Files.write( customLineByLineExtractorConfigFile.toPath()
- , customLineByLineExtractorConfig.replace("%EXTRACTOR_CLASS%", CSVExtractor.class.getName()).getBytes()
+ , customLineByLineExtractorConfig.replace("%EXTRACTOR_CLASS%", CSVExtractor.class.getName()).getBytes(
+ StandardCharsets.UTF_8)
, StandardOpenOption.CREATE_NEW , StandardOpenOption.TRUNCATE_EXISTING
);
if(file1.exists()) {
file1.delete();
}
Files.write( file1.toPath()
- , "google1.com,1,foo2\n".getBytes()
+ , "google1.com,1,foo2\n".getBytes(StandardCharsets.UTF_8)
, StandardOpenOption.CREATE_NEW , StandardOpenOption.TRUNCATE_EXISTING
);
if(file2.exists()) {
file2.delete();
}
Files.write( file2.toPath()
- , "google2.com,2,foo2\n".getBytes()
+ , "google2.com,2,foo2\n".getBytes(StandardCharsets.UTF_8)
, StandardOpenOption.CREATE_NEW , StandardOpenOption.TRUNCATE_EXISTING
);
@@ -256,10 +260,11 @@ public class SimpleEnrichmentFlatFileLoaderIntegrationTest {
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(multilineZipFile));
ZipEntry entry = new ZipEntry("file");
zos.putNextEntry(entry);
+ // Use intermediate OutputStreamWriters to specify encoding, instead of platform default.
pws = new PrintWriter[]{
- new PrintWriter(multilineFile),
- new PrintWriter(zos),
- new PrintWriter(new GZIPOutputStream(new FileOutputStream(multilineGzFile)))
+ new PrintWriter(multilineFile, StandardCharsets.UTF_8.name()),
+ new PrintWriter(new OutputStreamWriter(zos, StandardCharsets.UTF_8)),
+ new PrintWriter(new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(multilineGzFile)), StandardCharsets.UTF_8))
};
for(int i = 0;i < NUM_LINES;++i) {
for(PrintWriter pw : pws) {
@@ -282,7 +287,7 @@ public class SimpleEnrichmentFlatFileLoaderIntegrationTest {
private static void setupGlobalConfig(String zookeeperUrl) throws Exception {
client = ConfigurationsUtils.getClient(zookeeperUrl);
client.start();
- ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig.getBytes(), zookeeperUrl);
+ ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig.getBytes(StandardCharsets.UTF_8), zookeeperUrl);
}
@AfterClass
@@ -425,7 +430,8 @@ public class SimpleEnrichmentFlatFileLoaderIntegrationTest {
, "-p 2", "-b 128", "-q"
};
FileSystem fs = FileSystem.get(config);
- HBaseUtil.INSTANCE.writeFile(new String(Files.readAllBytes(multilineFile.toPath())), new Path(multilineFile.getName()), fs);
+ HBaseUtil.INSTANCE.writeFile(new String(Files.readAllBytes(multilineFile.toPath()),
+ StandardCharsets.UTF_8), new Path(multilineFile.getName()), fs);
SimpleEnrichmentFlatFileLoader.main(config, argv);
EnrichmentConverter converter = new EnrichmentConverter();
ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleFlatFileSummarizerTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleFlatFileSummarizerTest.java
index 17e3206..2b6f8e4 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleFlatFileSummarizerTest.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleFlatFileSummarizerTest.java
@@ -20,6 +20,7 @@ package org.apache.metron.dataloads.nonbulk.flatfile;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.PosixParser;
@@ -172,7 +173,7 @@ public class SimpleFlatFileSummarizerTest {
@Override
public InputStream openInputStream(String loc) throws IOException {
- return new ByteArrayInputStream(inMemoryData.get(loc).getBytes());
+ return new ByteArrayInputStream(inMemoryData.get(loc).getBytes(StandardCharsets.UTF_8));
}
@Override
diff --git a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java
index 77e0d61..abd2d18 100644
--- a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java
+++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java
@@ -20,9 +20,14 @@ package org.apache.metron.elasticsearch.bulk;
import java.io.BufferedReader;
import java.io.BufferedWriter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
@@ -66,7 +71,8 @@ public class ElasticsearchImportExport {
*/
public void bulkify(Path input, Path output) throws IOException {
List<String> outRecs = new ArrayList<String>();
- try (BufferedReader br = new BufferedReader(new FileReader(input.toFile()))) {
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(input.toFile()),
+ StandardCharsets.UTF_8))) {
String line;
while ((line = br.readLine()) != null) {
Map<String, Object> inDoc = JSONUtils.INSTANCE
@@ -80,7 +86,7 @@ public class ElasticsearchImportExport {
outRecs.add(outData);
}
}
- try (BufferedWriter br = new BufferedWriter(new FileWriter(output.toFile()))) {
+ try (BufferedWriter br = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output.toFile()), StandardCharsets.UTF_8))) {
for (String line : outRecs) {
br.write(line);
br.write(System.lineSeparator());
diff --git a/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
index b5bc20b..913d89c 100644
--- a/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
@@ -21,6 +21,7 @@ package org.apache.metron.enrichment.adapters.cif;
import java.io.IOException;
import java.io.Serializable;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
@@ -73,7 +74,7 @@ public class CIFHbaseAdapter implements EnrichmentAdapter<CacheKey>,Serializable
LOGGER.debug("=======Pinging HBase For: {}", key);
- Get get = new Get(key.getBytes());
+ Get get = new Get(key.getBytes(StandardCharsets.UTF_8));
Result rs;
Map output = new HashMap();
@@ -81,7 +82,7 @@ public class CIFHbaseAdapter implements EnrichmentAdapter<CacheKey>,Serializable
rs = table.get(get);
for (KeyValue kv : rs.raw())
- output.put(new String(kv.getQualifier()), "Y");
+ output.put(new String(kv.getQualifier(), StandardCharsets.UTF_8), "Y");
} catch (IOException e) {
// TODO Auto-generated catch block
diff --git a/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java
index 2e5e8bf..e96ea39 100644
--- a/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java
@@ -20,6 +20,7 @@ package org.apache.metron.enrichment.cli;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.cli.*;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
import org.apache.metron.common.utils.JSONUtils;
@@ -178,7 +179,8 @@ public class LatencySummarizer {
System.exit(0);
}
LatencyStats statsMap = new LatencyStats();
- BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(System.in,
+ StandardCharsets.UTF_8));
for(String line = null;(line = reader.readLine()) != null;) {
Map<String, Object> doc = JSONUtils.INSTANCE.load(line, JSONUtils.MAP_SUPPLIER);
updateStats(statsMap, doc);
diff --git a/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
index a138c0d..64f9e43 100644
--- a/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -17,11 +17,7 @@
*/
package org.apache.metron.enrichment.integration;
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.base.Splitter;
+import com.google.common.base.*;
import com.google.common.collect.Iterables;
import org.apache.commons.lang3.StringUtils;
import org.apache.metron.TestConstants;
@@ -57,14 +53,9 @@ import org.junit.Test;
import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
+import java.nio.charset.StandardCharsets;
import java.util.Objects;
-import java.util.Properties;
-import java.util.Set;
+import java.util.*;
import java.util.stream.Stream;
/**
@@ -104,7 +95,8 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
List<byte[]> ret = TestUtils.readSampleData(path);
{
//we want one of the fields without a destination IP to ensure that enrichments can function
- Map<String, Object> sansDestinationIp = JSONUtils.INSTANCE.load(new String(ret.get(ret.size() -1))
+ Map<String, Object> sansDestinationIp = JSONUtils.INSTANCE.load(new String(ret.get(ret.size() -1),
+ StandardCharsets.UTF_8)
, JSONUtils.MAP_SUPPLIER);
sansDestinationIp.remove(Constants.Fields.DST_ADDR.getName());
ret.add(JSONUtils.INSTANCE.toJSONPretty(sansDestinationIp));
@@ -567,7 +559,8 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
, Iterables.transform(outputMessages
, message -> {
try {
- return new HashMap<>(JSONUtils.INSTANCE.load(new String(message)
+ return new HashMap<>(JSONUtils.INSTANCE.load(new String(message,
+ StandardCharsets.UTF_8)
, JSONUtils.MAP_SUPPLIER
)
);
diff --git a/metron-platform/metron-hbase-server/src/test/java/org/apache/metron/hbase/coprocessor/EnrichmentCoprocessorTest.java b/metron-platform/metron-hbase-server/src/test/java/org/apache/metron/hbase/coprocessor/EnrichmentCoprocessorTest.java
index 19bd4b4..fdf7493 100644
--- a/metron-platform/metron-hbase-server/src/test/java/org/apache/metron/hbase/coprocessor/EnrichmentCoprocessorTest.java
+++ b/metron-platform/metron-hbase-server/src/test/java/org/apache/metron/hbase/coprocessor/EnrichmentCoprocessorTest.java
@@ -30,6 +30,7 @@ import static org.mockito.Mockito.when;
import com.github.benmanes.caffeine.cache.CacheWriter;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -145,7 +146,7 @@ public class EnrichmentCoprocessorTest {
thrown.expectMessage("Error occurred while processing enrichment Put.");
thrown.expectCause(instanceOf(RuntimeException.class));
cop.start(copEnv);
- cop.postPut(observerContext, new Put("foo".getBytes()), null, null);
+ cop.postPut(observerContext, new Put("foo".getBytes(StandardCharsets.UTF_8)), null, null);
}
@Test
diff --git a/metron-platform/metron-hbase/metron-hbase-common/src/main/java/org/apache/metron/hbase/HBaseProjectionCriteria.java b/metron-platform/metron-hbase/metron-hbase-common/src/main/java/org/apache/metron/hbase/HBaseProjectionCriteria.java
index 7432b02..54b2f7e 100644
--- a/metron-platform/metron-hbase/metron-hbase-common/src/main/java/org/apache/metron/hbase/HBaseProjectionCriteria.java
+++ b/metron-platform/metron-hbase/metron-hbase-common/src/main/java/org/apache/metron/hbase/HBaseProjectionCriteria.java
@@ -23,6 +23,7 @@ package org.apache.metron.hbase;
import com.google.common.collect.Lists;
import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
import java.util.List;
/**
@@ -44,8 +45,8 @@ public class HBaseProjectionCriteria implements Serializable {
private byte[] qualifier;
public ColumnMetaData(String columnFamily, String qualifier) {
- this.columnFamily = columnFamily.getBytes();
- this.qualifier = qualifier.getBytes();
+ this.columnFamily = columnFamily.getBytes(StandardCharsets.UTF_8);
+ this.qualifier = qualifier.getBytes(StandardCharsets.UTF_8);
}
public byte[] getColumnFamily() {
@@ -68,7 +69,7 @@ public class HBaseProjectionCriteria implements Serializable {
* @return column family along with all of its columns from an HBase lookup
*/
public HBaseProjectionCriteria addColumnFamily(String columnFamily) {
- this.columnFamilies.add(columnFamily.getBytes());
+ this.columnFamilies.add(columnFamily.getBytes(StandardCharsets.UTF_8));
return this;
}
diff --git a/metron-platform/metron-hbase/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java b/metron-platform/metron-hbase/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java
index 484d0e8..738275d 100644
--- a/metron-platform/metron-hbase/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java
+++ b/metron-platform/metron-hbase/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java
@@ -23,6 +23,7 @@ import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
import com.google.protobuf.Service;
import com.google.protobuf.ServiceException;
+import java.nio.charset.StandardCharsets;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -264,7 +265,7 @@ public class MockHTable implements HTableInterface {
qualifiers = data.get(row).get(family).navigableKeySet();
for (byte[] qualifier : qualifiers){
if (qualifier == null)
- qualifier = "".getBytes();
+ qualifier = "".getBytes(StandardCharsets.UTF_8);
if (!data.get(row).containsKey(family) ||
!data.get(row).get(family).containsKey(qualifier) ||
data.get(row).get(family).get(qualifier).isEmpty())
diff --git a/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
index 71d0544..6b246d4 100644
--- a/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
+++ b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
@@ -23,6 +23,7 @@ import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -160,7 +161,7 @@ public class HBaseDao implements IndexDao {
}
try {
tableInterface = config.getTableProvider().getTable(HBaseConfiguration.create(), table);
- this.cf = cf.getBytes();
+ this.cf = cf.getBytes(StandardCharsets.UTF_8);
} catch (IOException e) {
throw new IllegalStateException("Unable to initialize HBaseDao: " + e.getMessage(), e);
}
@@ -209,7 +210,8 @@ public class HBaseDao implements IndexDao {
Map.Entry<byte[], byte[]> entry= columns.lastEntry();
Long ts = Bytes.toLong(entry.getKey());
if(entry.getValue()!= null) {
- Map<String, Object> json = JSONUtils.INSTANCE.load(new String(entry.getValue()),
+ Map<String, Object> json = JSONUtils.INSTANCE.load(new String(entry.getValue(),
+ StandardCharsets.UTF_8),
JSONUtils.MAP_SUPPLIER);
// Make sure comments are in the proper format
diff --git a/metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java b/metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
index 26e65ba..ece07ca 100644
--- a/metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
+++ b/metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
@@ -20,6 +20,7 @@ package org.apache.metron.indexing.integration;
import com.google.common.collect.Iterables;
import org.apache.metron.TestConstants;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.common.field.FieldNameConverter;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.integration.*;
@@ -70,7 +71,7 @@ public class HDFSIndexingIntegrationTest extends IndexingIntegrationTest {
if (f.getName().startsWith("enrichment") || f.getName().endsWith(".json")) {
List<byte[]> data = TestUtils.readSampleData(f.getPath());
Iterables.addAll(ret, Iterables.transform(data, bytes -> {
- String s = new String(bytes);
+ String s = new String(bytes, StandardCharsets.UTF_8);
try {
return JSONUtils.INSTANCE.load(s,JSONUtils.MAP_SUPPLIER);
} catch (IOException e) {
diff --git a/metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java b/metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
index b6f3a70..b8719b6 100644
--- a/metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
+++ b/metron-platform/metron-indexing/metron-indexing-storm/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
@@ -18,27 +18,13 @@
package org.apache.metron.indexing.integration;
-import static org.apache.metron.common.configuration.ConfigurationsUtils.getClient;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.curator.framework.CuratorFramework;
import org.apache.metron.TestConstants;
import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.field.FieldNameConverter;
import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.integration.BaseIntegrationTest;
-import org.apache.metron.integration.ComponentRunner;
-import org.apache.metron.integration.InMemoryComponent;
-import org.apache.metron.integration.Processor;
-import org.apache.metron.integration.ProcessorResult;
+import org.apache.metron.integration.*;
import org.apache.metron.integration.components.ConfigUploadComponent;
import org.apache.metron.integration.components.FluxTopologyComponent;
import org.apache.metron.integration.components.KafkaComponent;
@@ -48,6 +34,13 @@ import org.apache.zookeeper.KeeperException;
import org.junit.Assert;
import org.junit.Test;
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.metron.common.configuration.ConfigurationsUtils.getClient;
+
public abstract class IndexingIntegrationTest extends BaseIntegrationTest {
protected static final String ERROR_TOPIC = "indexing_error";
protected String sampleParsedPath = "../" + TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
@@ -82,7 +75,7 @@ public abstract class IndexingIntegrationTest extends BaseIntegrationTest {
}});
List<Map<String, Object>> inputDocs = new ArrayList<>();
for(byte[] b : inputMessages) {
- Map<String, Object> m = JSONUtils.INSTANCE.load(new String(b), JSONUtils.MAP_SUPPLIER);
+ Map<String, Object> m = JSONUtils.INSTANCE.load(new String(b, StandardCharsets.UTF_8), JSONUtils.MAP_SUPPLIER);
inputDocs.add(m);
}
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/ProcessorResult.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/ProcessorResult.java
index c309686..b707b67 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/ProcessorResult.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/ProcessorResult.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.integration;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
@@ -69,7 +70,7 @@ public class ProcessorResult<T> {
}
buffer.append(String.format("%d Errors", processErrors.size()));
for (byte[] outputMessage : processErrors) {
- buffer.append(new String(outputMessage));
+ buffer.append(new String(outputMessage, StandardCharsets.UTF_8));
}
buffer.append("\n");
}
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java
index 6babb37..1a1ceb0 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java
@@ -18,10 +18,14 @@
package org.apache.metron.integration.components;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.InvocationTargetException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -267,7 +271,7 @@ public class FluxTopologyComponent implements InMemoryComponent {
File tmpFile = File.createTempFile(topologyName, "props");
tmpFile.deleteOnExit();
if (templateFile != null) {
- try (FileWriter propWriter = new FileWriter(tmpFile)){
+ try (Writer propWriter = new OutputStreamWriter(new FileOutputStream(tmpFile), StandardCharsets.UTF_8)){
String templateContents = FileUtils.readFileToString(templateFile);
for(Map.Entry prop: properties.entrySet()) {
String replacePattern = String.format("{{%s}}", prop.getKey());
@@ -278,7 +282,7 @@ public class FluxTopologyComponent implements InMemoryComponent {
return FluxParser.parseFile(yamlFile.getAbsolutePath(), false, true, tmpFile.getAbsolutePath(), false);
}
} else {
- try (FileWriter propWriter = new FileWriter(tmpFile)){
+ try (Writer propWriter = new OutputStreamWriter(new FileOutputStream(tmpFile), StandardCharsets.UTF_8)){
properties.store(propWriter, topologyName + " properties");
return FluxParser.parseFile(yamlFile.getAbsolutePath(), false, true, tmpFile.getAbsolutePath(), false);
}
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java
index 026c9fb..0c11bc5 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java
@@ -19,8 +19,10 @@ package org.apache.metron.integration.utils;
import java.io.BufferedReader;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
@@ -58,10 +60,10 @@ public class TestUtils {
}
public static List<byte[]> readSampleData(String samplePath) throws IOException {
- BufferedReader br = new BufferedReader(new FileReader(samplePath));
+ BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(samplePath), StandardCharsets.UTF_8));
List<byte[]> ret = new ArrayList<>();
for (String line = null; (line = br.readLine()) != null; ) {
- ret.add(line.getBytes());
+ ret.add(line.getBytes(StandardCharsets.UTF_8));
}
br.close();
return ret;
diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java
index 5a1281c..72ddad3 100644
--- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java
+++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java
@@ -18,6 +18,7 @@
package org.apache.metron.management;
import com.fasterxml.jackson.core.JsonProcessingException;
+import java.nio.charset.StandardCharsets;
import org.apache.curator.framework.CuratorFramework;
import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.EnrichmentConfigurations;
@@ -346,22 +347,22 @@ public class ConfigurationFunctions {
try {
if(GLOBAL == type) {
- writeGlobalConfigToZookeeper(value.getBytes(), client);
+ writeGlobalConfigToZookeeper(value.getBytes(StandardCharsets.UTF_8), client);
} else if(PROFILER == type) {
- writeProfilerConfigToZookeeper(value.getBytes(), client);
+ writeProfilerConfigToZookeeper(value.getBytes(StandardCharsets.UTF_8), client);
} else if(ENRICHMENT == type) {
String sensor = getArg(2, String.class, args);
- writeSensorEnrichmentConfigToZookeeper(sensor, value.getBytes(), client);
+ writeSensorEnrichmentConfigToZookeeper(sensor, value.getBytes(StandardCharsets.UTF_8), client);
} else if(INDEXING == type) {
String sensor = getArg(2, String.class, args);
- writeSensorIndexingConfigToZookeeper(sensor, value.getBytes(), client);
+ writeSensorIndexingConfigToZookeeper(sensor, value.getBytes(StandardCharsets.UTF_8), client);
} else if (PARSER == type) {
String sensor = getArg(2, String.class, args);
- writeSensorParserConfigToZookeeper(sensor, value.getBytes(), client);
+ writeSensorParserConfigToZookeeper(sensor, value.getBytes(StandardCharsets.UTF_8), client);
}
} catch(Exception e) {
diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/GrokFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/GrokFunctions.java
index c07eb10..deb1955 100644
--- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/GrokFunctions.java
+++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/GrokFunctions.java
@@ -22,6 +22,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -44,7 +45,7 @@ public class GrokFunctions {
InputStream input = GrokFunctions.class.getResourceAsStream("/patterns/common");
if(input != null) {
- grok.addPatternFromReader(new InputStreamReader(input));
+ grok.addPatternFromReader(new InputStreamReader(input, StandardCharsets.UTF_8));
}
if(grokExpr != null) {
diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/StellarParserRunner.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/StellarParserRunner.java
index 8884393..856f718 100644
--- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/StellarParserRunner.java
+++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/StellarParserRunner.java
@@ -18,6 +18,7 @@
package org.apache.metron.management;
import com.fasterxml.jackson.core.JsonProcessingException;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.parsers.ParserRunnerImpl;
@@ -76,7 +77,7 @@ public class StellarParserRunner {
// parse each message
List<ParserRunnerResults<JSONObject>> results = messages
.stream()
- .map(str -> str.getBytes())
+ .map(str -> str.getBytes(StandardCharsets.UTF_8))
.map(bytes -> DEFAULT.get(emptyMap(), bytes, false, emptyMap()))
.map(msg -> runner.execute(sensorType, msg, parserConfigurations))
.collect(Collectors.toList());
@@ -101,12 +102,13 @@ public class StellarParserRunner {
}
public StellarParserRunner withParserConfiguration(String sensorConfig) {
- parserConfigurations = create(sensorConfig.getBytes());
+ parserConfigurations = create(sensorConfig.getBytes(StandardCharsets.UTF_8));
return this;
}
public StellarParserRunner withParserConfiguration(Map<String, Object> config) {
- parserConfigurations = create(new JSONObject(config).toJSONString().getBytes());
+ parserConfigurations = create(new JSONObject(config).toJSONString().getBytes(
+ StandardCharsets.UTF_8));
return this;
}
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
index 67e2a9d..d7d6515 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
@@ -18,6 +18,7 @@
package org.apache.metron.management;
import com.google.common.collect.ImmutableMap;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.PosixParser;
@@ -110,7 +111,7 @@ public class ConfigurationFunctionsTest {
// push configs to zookeeper
pushConfigs(SAMPLE_CONFIG_PATH, zookeeperUrl);
pushConfigs(PARSER_CONFIGS_PATH, zookeeperUrl);
- writeProfilerConfigToZookeeper(goodProfilerConfig.getBytes(), client);
+ writeProfilerConfigToZookeeper(goodProfilerConfig.getBytes(StandardCharsets.UTF_8), client);
}
/**
@@ -167,8 +168,9 @@ public class ConfigurationFunctionsTest {
String out = (String) run("CONFIG_GET('PARSER', 'bro')", context);
- SensorParserConfig actual = SensorParserConfig.fromBytes(out.getBytes());
- SensorParserConfig expected = SensorParserConfig.fromBytes(goodBroParserConfig.getBytes());
+ SensorParserConfig actual = SensorParserConfig.fromBytes(out.getBytes(StandardCharsets.UTF_8));
+ SensorParserConfig expected = SensorParserConfig.fromBytes(goodBroParserConfig.getBytes(
+ StandardCharsets.UTF_8));
assertEquals(expected, actual);
}
@@ -194,12 +196,14 @@ public class ConfigurationFunctionsTest {
SensorParserConfig expected = new SensorParserConfig();
{
Object out = run("CONFIG_GET('PARSER', 'sensor')", context);
- SensorParserConfig actual = SensorParserConfig.fromBytes(out.toString().getBytes());
+ SensorParserConfig actual = SensorParserConfig.fromBytes(out.toString().getBytes(
+ StandardCharsets.UTF_8));
assertEquals(expected, actual);
}
{
Object out = run("CONFIG_GET('PARSER', 'sensor', true)", context);
- SensorParserConfig actual = SensorParserConfig.fromBytes(out.toString().getBytes());
+ SensorParserConfig actual = SensorParserConfig.fromBytes(out.toString().getBytes(
+ StandardCharsets.UTF_8));
assertEquals(expected, actual);
}
}
@@ -213,8 +217,10 @@ public class ConfigurationFunctionsTest {
String out = (String) run("CONFIG_GET('ENRICHMENT', 'test')", context);
- SensorEnrichmentConfig actual = SensorEnrichmentConfig.fromBytes(out.getBytes());
- SensorEnrichmentConfig expected = SensorEnrichmentConfig.fromBytes(goodTestEnrichmentConfig.getBytes());
+ SensorEnrichmentConfig actual = SensorEnrichmentConfig.fromBytes(out.getBytes(
+ StandardCharsets.UTF_8));
+ SensorEnrichmentConfig expected = SensorEnrichmentConfig.fromBytes(goodTestEnrichmentConfig.getBytes(
+ StandardCharsets.UTF_8));
assertEquals(expected, actual);
}
@@ -239,12 +245,14 @@ public class ConfigurationFunctionsTest {
SensorEnrichmentConfig expected = new SensorEnrichmentConfig();
{
String out = (String) run("CONFIG_GET('ENRICHMENT', 'missing-sensor')", context);
- SensorEnrichmentConfig actual = SensorEnrichmentConfig.fromBytes(out.getBytes());
+ SensorEnrichmentConfig actual = SensorEnrichmentConfig.fromBytes(out.getBytes(
+ StandardCharsets.UTF_8));
assertEquals(expected, actual);
}
{
String out = (String) run("CONFIG_GET('ENRICHMENT', 'missing-sensor', true)", context);
- SensorEnrichmentConfig actual = SensorEnrichmentConfig.fromBytes(out.getBytes());
+ SensorEnrichmentConfig actual = SensorEnrichmentConfig.fromBytes(out.getBytes(
+ StandardCharsets.UTF_8));
assertEquals(expected, actual);
}
}
@@ -302,8 +310,9 @@ public class ConfigurationFunctionsTest {
String out = (String) run("CONFIG_GET('PROFILER')", context);
- ProfilerConfig actual = ProfilerConfig.fromBytes(out.getBytes());
- ProfilerConfig expected = ProfilerConfig.fromBytes(goodProfilerConfig.getBytes());
+ ProfilerConfig actual = ProfilerConfig.fromBytes(out.getBytes(StandardCharsets.UTF_8));
+ ProfilerConfig expected = ProfilerConfig.fromBytes(goodProfilerConfig.getBytes(
+ StandardCharsets.UTF_8));
assertEquals(expected, actual);
}
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserFunctionsTest.java
index 5510c67..807f47d 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserFunctionsTest.java
@@ -19,6 +19,7 @@
package org.apache.metron.management;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.GetDataBuilder;
@@ -220,7 +221,7 @@ public class ParserFunctionsTest {
String config = execute("PARSER_CONFIG(parser)", String.class);
Assert.assertNotNull(config);
- Assert.assertNotNull(SensorParserConfig.fromBytes(config.getBytes()));
+ Assert.assertNotNull(SensorParserConfig.fromBytes(config.getBytes(StandardCharsets.UTF_8)));
}
@Test
@@ -230,7 +231,8 @@ public class ParserFunctionsTest {
Assert.assertNotNull(runner);
SensorParserConfig actual = runner.getParserConfigurations().getSensorParserConfig("bro");
- SensorParserConfig expected = SensorParserConfig.fromBytes(broParserConfig.getBytes());
+ SensorParserConfig expected = SensorParserConfig.fromBytes(broParserConfig.getBytes(
+ StandardCharsets.UTF_8));
Assert.assertEquals(expected, actual);
}
@@ -242,7 +244,8 @@ public class ParserFunctionsTest {
Assert.assertNotNull(runner);
SensorParserConfig actual = runner.getParserConfigurations().getSensorParserConfig("bro");
- SensorParserConfig expected = SensorParserConfig.fromBytes(broParserConfig.getBytes());
+ SensorParserConfig expected = SensorParserConfig.fromBytes(broParserConfig.getBytes(
+ StandardCharsets.UTF_8));
Assert.assertEquals(expected, actual);
}
@@ -254,7 +257,7 @@ public class ParserFunctionsTest {
@Test
public void testInitFromZookeeper() throws Exception {
- byte[] configAsBytes = broParserConfig.getBytes();
+ byte[] configAsBytes = broParserConfig.getBytes(StandardCharsets.UTF_8);
CuratorFramework zkClient = zkClientForPath("/metron/topology/parsers/bro", configAsBytes);
context.addCapability(Context.Capabilities.ZOOKEEPER_CLIENT, () -> zkClient);
@@ -262,7 +265,8 @@ public class ParserFunctionsTest {
Assert.assertNotNull(runner);
SensorParserConfig actual = runner.getParserConfigurations().getSensorParserConfig("bro");
- SensorParserConfig expected = SensorParserConfig.fromBytes(broParserConfig.getBytes());
+ SensorParserConfig expected = SensorParserConfig.fromBytes(broParserConfig.getBytes(
+ StandardCharsets.UTF_8));
Assert.assertEquals(expected, actual);
}
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/GrokParser.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/GrokParser.java
index 12fa8e2..14873e3 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/GrokParser.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/GrokParser.java
@@ -116,7 +116,7 @@ public class GrokParser implements MessageParser<JSONObject>, Serializable {
"Unable to initialize grok parser: Unable to load " + patternsCommonDir + " from either classpath or HDFS");
}
- grok.addPatternFromReader(new InputStreamReader(commonInputStream));
+ grok.addPatternFromReader(new InputStreamReader(commonInputStream, StandardCharsets.UTF_8));
LOG.info("Loading parser-specific patterns from: {}", grokPath);
InputStream patterInputStream = openInputStream(grokPath);
@@ -124,7 +124,7 @@ public class GrokParser implements MessageParser<JSONObject>, Serializable {
throw new RuntimeException("Grok parser unable to initialize grok parser: Unable to load " + grokPath
+ " from either classpath or HDFS");
}
- grok.addPatternFromReader(new InputStreamReader(patterInputStream));
+ grok.addPatternFromReader(new InputStreamReader(patterInputStream, StandardCharsets.UTF_8));
LOG.info("Grok parser set the following grok expression for '{}': {}", () ->patternLabel,
() -> grok.getPatterns().get(patternLabel));
@@ -211,7 +211,7 @@ public class GrokParser implements MessageParser<JSONObject>, Serializable {
Map<Object,Throwable> errors = new HashMap<>();
String originalMessage = null;
try {
- originalMessage = new String(rawMessage, "UTF-8");
+ originalMessage = new String(rawMessage, StandardCharsets.UTF_8);
LOG.debug("Grok parser parsing message: {}",originalMessage);
Match gm = grok.match(originalMessage);
gm.captures();
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java
index cb6c0c4..d76205d 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java
@@ -19,6 +19,7 @@ package org.apache.metron.parsers;
import java.io.Serializable;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -255,7 +256,7 @@ public class ParserRunnerImpl implements ParserRunner<JSONObject>, Serializable
if (!message.containsKey(Constants.GUID)) {
message.put(Constants.GUID, UUID.randomUUID().toString());
}
- message.putIfAbsent(Fields.ORIGINAL.getName(), new String(rawMessage.getMessage()));
+ message.putIfAbsent(Fields.ORIGINAL.getName(), new String(rawMessage.getMessage(), StandardCharsets.UTF_8));
MessageFilter<JSONObject> filter = sensorToParserComponentMap.get(sensorType).getFilter();
if (filter == null || filter.emit(message, stellarContext)) {
boolean isInvalid = !parser.validate(message);
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/csv/CSVParser.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/csv/CSVParser.java
index 444f021..35e23d3 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/csv/CSVParser.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/csv/CSVParser.java
@@ -20,6 +20,7 @@ package org.apache.metron.parsers.csv;
import com.google.common.collect.ImmutableList;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.List;
@@ -55,7 +56,7 @@ public class CSVParser extends BasicParser {
@Override
public List<JSONObject> parse(byte[] rawMessage) {
try {
- String msg = new String(rawMessage, "UTF-8");
+ String msg = new String(rawMessage, StandardCharsets.UTF_8);
Map<String, String> value = converter.toMap(msg);
if(value != null) {
value.put("original_string", msg);
@@ -87,7 +88,7 @@ public class CSVParser extends BasicParser {
return Collections.emptyList();
}
} catch (Throwable e) {
- String message = "Unable to parse " + new String(rawMessage) + ": " + e.getMessage();
+ String message = "Unable to parse " + new String(rawMessage, StandardCharsets.UTF_8) + ": " + e.getMessage();
LOG.error(message, e);
throw new IllegalStateException(message, e);
}
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
index bb9bb54..5da7c05 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
@@ -29,6 +29,7 @@ import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
import com.jayway.jsonpath.spi.json.JsonProvider;
import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
import com.jayway.jsonpath.spi.mapper.MappingProvider;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
@@ -169,7 +170,7 @@ public class JSONMapParser extends BasicParser {
@SuppressWarnings("unchecked")
public List<JSONObject> parse(byte[] rawMessage) {
try {
- String rawString = new String(rawMessage);
+ String rawString = new String(rawMessage, StandardCharsets.UTF_8);
List<Map<String, Object>> messages = new ArrayList<>();
// if configured, wrap the json in an entity and array
@@ -202,7 +203,7 @@ public class JSONMapParser extends BasicParser {
}
return Collections.unmodifiableList(parsedMessages);
} catch (Throwable e) {
- String message = "Unable to parse " + new String(rawMessage) + ": " + e.getMessage();
+ String message = "Unable to parse " + new String(rawMessage, StandardCharsets.UTF_8) + ": " + e.getMessage();
LOG.error(message, e);
throw new IllegalStateException(message, e);
}
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/regex/RegularExpressionsParser.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/regex/RegularExpressionsParser.java
index c9f1ec9..d375451 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/regex/RegularExpressionsParser.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/regex/RegularExpressionsParser.java
@@ -25,6 +25,7 @@ import org.slf4j.LoggerFactory;
import java.lang.invoke.MethodHandles;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -132,8 +133,6 @@ public class RegularExpressionsParser extends BasicParser {
protected static final Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static final Charset UTF_8 = Charset.forName("UTF-8");
-
private List<Map<String, Object>> fields;
private Map<String, Object> parserConfig;
private final Pattern namedGroupPattern = Pattern.compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>");
@@ -155,7 +154,7 @@ public class RegularExpressionsParser extends BasicParser {
public List<JSONObject> parse(byte[] rawMessage) {
String originalMessage = null;
try {
- originalMessage = new String(rawMessage, UTF_8).trim();
+ originalMessage = new String(rawMessage, StandardCharsets.UTF_8).trim();
LOG.debug(" raw message. {}", originalMessage);
if (originalMessage.isEmpty()) {
LOG.warn("Message is empty.");
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/syslog/BaseSyslogParser.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/syslog/BaseSyslogParser.java
index c05b760..c349fd8 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/syslog/BaseSyslogParser.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/syslog/BaseSyslogParser.java
@@ -20,6 +20,7 @@ package org.apache.metron.parsers.syslog;
import com.github.palindromicity.syslog.SyslogParser;
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.lang3.StringUtils;
import org.apache.metron.parsers.DefaultMessageParserResult;
import org.apache.metron.parsers.ParseException;
@@ -109,7 +110,7 @@ public abstract class BaseSyslogParser implements MessageParser<JSONObject>, Ser
return Optional.empty();
}
- String originalString = new String(rawMessage);
+ String originalString = new String(rawMessage, StandardCharsets.UTF_8);
final List<JSONObject> returnList = new ArrayList<>();
Map<Object,Throwable> errorMap = new HashMap<>();
try (Reader reader = new BufferedReader(new StringReader(originalString))) {
@@ -131,7 +132,7 @@ public abstract class BaseSyslogParser implements MessageParser<JSONObject>, Ser
return Optional.of(new DefaultMessageParserResult<JSONObject>(returnList,errorMap));
}
} catch (IOException e) {
- String message = "Unable to read buffer " + new String(rawMessage) + ": " + e.getMessage();
+ String message = "Unable to read buffer " + new String(rawMessage, StandardCharsets.UTF_8) + ": " + e.getMessage();
LOG.error(message, e);
return Optional.of(new DefaultMessageParserResult<JSONObject>( new IllegalStateException(message, e)));
}
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/AbstractParserConfigTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/AbstractParserConfigTest.java
index dd43cc3..f9ec14e 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/AbstractParserConfigTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/AbstractParserConfigTest.java
@@ -25,9 +25,12 @@ import com.github.fge.jsonschema.main.JsonSchemaFactory;
import com.github.fge.jsonschema.main.JsonValidator;
import java.io.BufferedReader;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
public class AbstractParserConfigTest {
@@ -51,8 +54,8 @@ public class AbstractParserConfigTest {
}
protected String readSchemaFromFile(URL schema_url) throws Exception {
- BufferedReader br = new BufferedReader(new FileReader(
- schema_url.getFile()));
+ BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(schema_url.getFile()),
+ StandardCharsets.UTF_8));
String line;
StringBuilder sb = new StringBuilder();
while ((line = br.readLine()) != null) {
@@ -67,7 +70,7 @@ public class AbstractParserConfigTest {
}
protected String[] readTestDataFromFile(String test_data_url) throws Exception {
- BufferedReader br = new BufferedReader(new FileReader(new File(test_data_url)));
+ BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(test_data_url), StandardCharsets.UTF_8));
ArrayList<String> inputDataLines = new ArrayList<>();
String line;
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/GrokParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/GrokParserTest.java
index 24386fa..352de0a 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/GrokParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/GrokParserTest.java
@@ -19,6 +19,7 @@ package org.apache.metron.parsers;
import com.google.common.collect.MapDifference;
import com.google.common.collect.Maps;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.parsers.interfaces.MessageParserResult;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
@@ -54,7 +55,7 @@ public abstract class GrokParserTest {
for( Map.Entry<String,String> e : testData.entrySet() ){
JSONObject expected = (JSONObject) jsonParser.parse(e.getValue());
- byte[] rawMessage = e.getKey().getBytes();
+ byte[] rawMessage = e.getKey().getBytes(StandardCharsets.UTF_8);
Optional<MessageParserResult<JSONObject>> resultOptional = grokParser.parseOptionalResult(rawMessage);
Assert.assertNotNull(resultOptional);
Assert.assertTrue(resultOptional.isPresent());
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MessageParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MessageParserTest.java
index 4842a1f..1d4de5b 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MessageParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MessageParserTest.java
@@ -18,6 +18,7 @@
package org.apache.metron.parsers;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.interfaces.MessageParserResult;
import org.json.simple.JSONObject;
@@ -84,7 +85,8 @@ public class MessageParserTest {
return Collections.singletonList(message);
}
};
- Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes());
+ Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes(
+ StandardCharsets.UTF_8));
Assert.assertTrue(ret.isPresent());
Assert.assertEquals(1, ret.get().getMessages().size());
Assert.assertEquals(message, ret.get().getMessages().get(0));
@@ -99,7 +101,8 @@ public class MessageParserTest {
return Optional.of(Collections.singletonList(message));
}
};
- Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes());
+ Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes(
+ StandardCharsets.UTF_8));
Assert.assertTrue(ret.isPresent());
Assert.assertEquals(1, ret.get().getMessages().size());
Assert.assertEquals(message, ret.get().getMessages().get(0));
@@ -113,7 +116,8 @@ public class MessageParserTest {
throw new RuntimeException("parse exception");
}
};
- Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes());
+ Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes(
+ StandardCharsets.UTF_8));
Assert.assertTrue(ret.isPresent());
Assert.assertTrue(ret.get().getMasterThrowable().isPresent());
Assert.assertEquals("parse exception", ret.get().getMasterThrowable().get().getMessage());
@@ -127,7 +131,8 @@ public class MessageParserTest {
throw new RuntimeException("parse exception");
}
};
- Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes());
+ Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes(
+ StandardCharsets.UTF_8));
Assert.assertTrue(ret.isPresent());
Assert.assertTrue(ret.get().getMasterThrowable().isPresent());
Assert.assertEquals("parse exception", ret.get().getMasterThrowable().get().getMessage());
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
index 8478bd4..8d346c3 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.parsers;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.io.IOUtils;
import org.apache.metron.parsers.interfaces.MessageParserResult;
import org.json.simple.JSONObject;
@@ -60,7 +61,7 @@ public class MultiLineGrokParserTest {
JSONParser jsonParser = new JSONParser();
Map<String, String> testData = getTestData();
for (Map.Entry<String, String> e : testData.entrySet()) {
- byte[] rawMessage = e.getKey().getBytes();
+ byte[] rawMessage = e.getKey().getBytes(StandardCharsets.UTF_8);
Optional<MessageParserResult<JSONObject>> resultOptional = grokParser.parseOptionalResult(rawMessage);
Assert.assertNotNull(resultOptional);
Assert.assertTrue(resultOptional.isPresent());
@@ -95,7 +96,7 @@ public class MultiLineGrokParserTest {
JSONParser jsonParser = new JSONParser();
Map<String, String> testData = getTestData();
for (Map.Entry<String, String> e : testData.entrySet()) {
- byte[] rawMessage = e.getKey().getBytes();
+ byte[] rawMessage = e.getKey().getBytes(StandardCharsets.UTF_8);
Optional<MessageParserResult<JSONObject>> resultOptional = grokParser.parseOptionalResult(rawMessage);
Assert.assertTrue(resultOptional.isPresent());
Optional<Throwable> throwableOptional = resultOptional.get().getMasterThrowable();
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java
index 4a235bf..7ce0802 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.parsers;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.io.IOUtils;
import org.apache.metron.parsers.interfaces.MessageParserResult;
import org.json.simple.JSONObject;
@@ -61,7 +62,7 @@ public class MultiLineWithErrorsGrokParserTest {
JSONParser jsonParser = new JSONParser();
Map<String, String> testData = getTestData();
for (Map.Entry<String, String> e : testData.entrySet()) {
- byte[] rawMessage = e.getKey().getBytes();
+ byte[] rawMessage = e.getKey().getBytes(StandardCharsets.UTF_8);
List<JSONObject> parsedList = grokParser.parse(rawMessage);
}
}
@@ -92,7 +93,7 @@ public class MultiLineWithErrorsGrokParserTest {
JSONParser jsonParser = new JSONParser();
Map<String, String> testData = getTestData();
for (Map.Entry<String, String> e : testData.entrySet()) {
- byte[] rawMessage = e.getKey().getBytes();
+ byte[] rawMessage = e.getKey().getBytes(StandardCharsets.UTF_8);
Optional<MessageParserResult<JSONObject>> resultOptional = grokParser.parseOptionalResult(rawMessage);
Assert.assertTrue(resultOptional.isPresent());
Optional<Throwable> throwableOptional = resultOptional.get().getMasterThrowable();
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
index c0a85da..486cda3 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
@@ -28,6 +28,7 @@ import static org.powermock.api.mockito.PowerMockito.mockStatic;
import static org.powermock.api.mockito.PowerMockito.when;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -119,8 +120,10 @@ public class ParserRunnerImplTest {
@Before
public void setup() throws IOException {
parserConfigurations = new ParserConfigurations();
- SensorParserConfig broConfig = SensorParserConfig.fromBytes(broConfigString.getBytes());
- SensorParserConfig snortConfig = SensorParserConfig.fromBytes(snortConfigString.getBytes());
+ SensorParserConfig broConfig = SensorParserConfig.fromBytes(broConfigString.getBytes(
+ StandardCharsets.UTF_8));
+ SensorParserConfig snortConfig = SensorParserConfig.fromBytes(snortConfigString.getBytes(
+ StandardCharsets.UTF_8));
parserConfigurations.updateSensorParserConfig("bro", broConfig);
parserConfigurations.updateSensorParserConfig("snort", snortConfig);
parserConfigurations.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfigString, JSONUtils.MAP_SUPPLIER));
@@ -220,13 +223,13 @@ public class ParserRunnerImplTest {
@Test
public void shouldExecute() {
parserRunner = spy(parserRunner);
- RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>());
+ RawMessage rawMessage = new RawMessage("raw_message".getBytes(StandardCharsets.UTF_8), new HashMap<>());
JSONObject parsedMessage1 = new JSONObject();
parsedMessage1.put("field", "parsedMessage1");
JSONObject parsedMessage2 = new JSONObject();
parsedMessage2.put("field", "parsedMessage2");
- Object rawMessage1 = new RawMessage("raw_message1".getBytes(), new HashMap<>());
- Object rawMessage2 = new RawMessage("raw_message2".getBytes(), new HashMap<>());
+ Object rawMessage1 = new RawMessage("raw_message1".getBytes(StandardCharsets.UTF_8), new HashMap<>());
+ Object rawMessage2 = new RawMessage("raw_message2".getBytes(StandardCharsets.UTF_8), new HashMap<>());
Throwable throwable1 = mock(Throwable.class);
Throwable throwable2 = mock(Throwable.class);
MessageParserResult<JSONObject> messageParserResult = new DefaultMessageParserResult<>(Arrays.asList(parsedMessage1, parsedMessage2),
@@ -276,7 +279,7 @@ public class ParserRunnerImplTest {
@Test
public void shouldExecuteWithMasterThrowable() {
parserRunner = spy(parserRunner);
- RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>());
+ RawMessage rawMessage = new RawMessage("raw_message".getBytes(StandardCharsets.UTF_8), new HashMap<>());
Throwable masterThrowable = mock(Throwable.class);
MessageParserResult<JSONObject> messageParserResult = new DefaultMessageParserResult<>(masterThrowable);
@@ -309,7 +312,7 @@ public class ParserRunnerImplTest {
inputMessage.put("guid", "guid");
inputMessage.put("ip_src_addr", "192.168.1.1");
inputMessage.put("ip_dst_addr", "192.168.1.2");
- RawMessage rawMessage = new RawMessage("raw_message_for_testing".getBytes(), new HashMap<>());
+ RawMessage rawMessage = new RawMessage("raw_message".getBytes(StandardCharsets.UTF_8), new HashMap<>());
JSONObject expectedOutput = new JSONObject();
expectedOutput.put("guid", "guid");
@@ -341,7 +344,7 @@ public class ParserRunnerImplTest {
inputMessage.put("ip_src_addr", "192.168.1.1");
inputMessage.put("ip_dst_addr", "192.168.1.2");
inputMessage.put(Fields.ORIGINAL.getName(), "original_string_added_by_parser");
- RawMessage rawMessage = new RawMessage("raw_message_for_testing".getBytes(), new HashMap<>());
+ RawMessage rawMessage = new RawMessage("raw_message_for_testing".getBytes(StandardCharsets.UTF_8), new HashMap<>());
JSONObject expectedOutput = new JSONObject();
expectedOutput.put("guid", "guid");
@@ -372,7 +375,7 @@ public class ParserRunnerImplTest {
JSONObject inputMessage = new JSONObject();
inputMessage.put("guid", "guid");
- RawMessage rawMessage = new RawMessage("raw_message".getBytes(), metadata);
+ RawMessage rawMessage = new RawMessage("raw_message".getBytes(StandardCharsets.UTF_8), metadata);
JSONObject expectedOutput = new JSONObject();
expectedOutput.put("guid", "guid");
@@ -410,7 +413,7 @@ public class ParserRunnerImplTest {
inputMessage.put("guid", "guid");
inputMessage.put("ip_src_addr", "test");
inputMessage.put("ip_dst_addr", "test");
- RawMessage rawMessage = new RawMessage("raw_message".getBytes(), metadata);
+ RawMessage rawMessage = new RawMessage("raw_message".getBytes(StandardCharsets.UTF_8), metadata);
JSONObject expectedOutput = new JSONObject();
expectedOutput.put("guid", "guid");
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/EnvelopedParserIntegrationTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/EnvelopedParserIntegrationTest.java
index 3044061..46836a3 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/EnvelopedParserIntegrationTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/EnvelopedParserIntegrationTest.java
@@ -19,6 +19,7 @@ package org.apache.metron.parsers.integration;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -61,7 +62,8 @@ public abstract class EnvelopedParserIntegrationTest {
Assert.assertFalse(results.failed());
List<byte[]> resultList = results.getResult();
Assert.assertEquals(1, resultList.size());
- Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER);
+ Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0),
+ StandardCharsets.UTF_8), JSONUtils.MAP_SUPPLIER);
Assert.assertEquals("field1_val", outputRecord.get("field1"));
Assert.assertEquals(inputRecord.get(Constants.Fields.ORIGINAL.getName()), outputRecord.get(Constants.Fields.ORIGINAL.getName()));
Assert.assertEquals(inputRecord.get(MetadataUtil.METADATA_PREFIX + ".metadata_field"), outputRecord.get("metadata_field"));
@@ -98,7 +100,8 @@ public abstract class EnvelopedParserIntegrationTest {
Assert.assertFalse(results.failed());
List<byte[]> resultList = results.getResult();
Assert.assertEquals(1, resultList.size());
- Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER);
+ Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0),
+ StandardCharsets.UTF_8), JSONUtils.MAP_SUPPLIER);
Assert.assertEquals("field1_val", outputRecord.get("field1"));
Assert.assertEquals(inputRecord.get(Constants.Fields.ORIGINAL.getName()), outputRecord.get(Constants.Fields.ORIGINAL.getName()));
Assert.assertEquals(inputRecord.get("metadata_field"), outputRecord.get("metadata_field"));
@@ -134,7 +137,8 @@ public abstract class EnvelopedParserIntegrationTest {
Assert.assertFalse(results.failed());
List<byte[]> resultList = results.getResult();
Assert.assertEquals(1, resultList.size());
- Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER);
+ Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0),
+ StandardCharsets.UTF_8), JSONUtils.MAP_SUPPLIER);
Assert.assertEquals("field1_val", outputRecord.get("field1"));
Assert.assertEquals(inputRecord.get(Constants.Fields.ORIGINAL.getName()), outputRecord.get(Constants.Fields.ORIGINAL.getName()));
Assert.assertFalse(outputRecord.containsKey(MetadataUtil.METADATA_PREFIX + ".metadata_field"));
@@ -179,7 +183,8 @@ public abstract class EnvelopedParserIntegrationTest {
throws Exception {
byte[] envelopedData = null;
String inputRecord = "Mar 29 2004 09:54:18: %PIX-6-302005: Built UDP connection for faddr 198.207.223.240/53337 gaddr 10.0.0.187/53 laddr 192.168.0.2/53";
- ProcessorResult<List<byte[]>> syslogResult = syslogDriver.run(ImmutableList.of(inputRecord.getBytes()));
+ ProcessorResult<List<byte[]>> syslogResult = syslogDriver.run(ImmutableList.of(inputRecord.getBytes(
+ StandardCharsets.UTF_8)));
Assert.assertFalse(syslogResult.failed());
List<byte[]> syslogResultList = syslogResult.getResult();
envelopedData = syslogResultList.get(0);
@@ -187,7 +192,8 @@ public abstract class EnvelopedParserIntegrationTest {
Assert.assertFalse(results.failed());
List<byte[]> resultList = results.getResult();
Assert.assertEquals(1, resultList.size());
- Map<String, Object> result = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER);
+ Map<String, Object> result = JSONUtils.INSTANCE.load(new String(resultList.get(0),
+ StandardCharsets.UTF_8), JSONUtils.MAP_SUPPLIER);
Assert.assertEquals("UDP", result.get("protocol"));
Assert.assertTrue((long) result.get("timestamp") > 1000);
}
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
index 29ec1fe..09b007c 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
@@ -19,6 +19,7 @@ package org.apache.metron.parsers.integration;
import java.io.File;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.List;
@@ -53,7 +54,8 @@ public abstract class ParserIntegrationTest extends BaseIntegrationTest {
protected String readGlobalConfig() throws IOException {
File configsRoot = new File("../" + TestConstants.SAMPLE_CONFIG_PATH);
- return new String(Files.readAllBytes(new File(configsRoot, "global.json").toPath()));
+ return new String(Files.readAllBytes(new File(configsRoot, "global.json").toPath()),
+ StandardCharsets.UTF_8);
}
protected String readSensorConfig(String sensorType) throws IOException {
@@ -67,7 +69,8 @@ public abstract class ParserIntegrationTest extends BaseIntegrationTest {
configsRoot = new File("../" + TestConstants.PARSER_CONFIGS_PATH);
parsersRoot = new File(configsRoot, "parsers");
}
- return new String(Files.readAllBytes(new File(parsersRoot, sensorType + ".json").toPath()));
+ return new String(Files.readAllBytes(new File(parsersRoot, sensorType + ".json").toPath()),
+ StandardCharsets.UTF_8);
}
public void runTest(ParserDriver driver) throws Exception {
@@ -99,7 +102,7 @@ public abstract class ParserIntegrationTest extends BaseIntegrationTest {
public void dumpParsedMessages(List<byte[]> outputMessages, StringBuffer buffer) {
for (byte[] outputMessage : outputMessages) {
- buffer.append(new String(outputMessage)).append("\n");
+ buffer.append(new String(outputMessage, StandardCharsets.UTF_8)).append("\n");
}
}
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/ParserDriver.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/ParserDriver.java
index 4854429..f830460 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/ParserDriver.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/ParserDriver.java
@@ -19,6 +19,7 @@ package org.apache.metron.parsers.integration.validation;
import java.io.IOException;
import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
import java.util.HashSet;
import java.util.List;
import org.apache.metron.common.configuration.ParserConfigurations;
@@ -40,11 +41,12 @@ public abstract class ParserDriver implements Serializable {
public ParserDriver(String sensorType, String parserConfig, String globalConfig)
throws IOException {
- SensorParserConfig sensorParserConfig = SensorParserConfig.fromBytes(parserConfig.getBytes());
+ SensorParserConfig sensorParserConfig = SensorParserConfig.fromBytes(parserConfig.getBytes(
+ StandardCharsets.UTF_8));
this.sensorType = sensorType == null ? sensorParserConfig.getSensorTopic() : sensorType;
config = new ParserConfigurations();
config.updateSensorParserConfig(this.sensorType,
- SensorParserConfig.fromBytes(parserConfig.getBytes()));
+ SensorParserConfig.fromBytes(parserConfig.getBytes(StandardCharsets.UTF_8)));
config.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER));
parserRunner = new ParserRunnerImpl(new HashSet<String>() {{
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java
index adc246d..44e0d4d 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.parsers.integration.validation;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.integration.utils.TestUtils;
import org.apache.metron.parsers.integration.ParserValidation;
import org.apache.metron.test.TestDataType;
@@ -38,8 +39,8 @@ public class SampleDataValidation implements ParserValidation {
List<byte[]> expectedMessages = TestUtils.readSampleData(SampleDataUtils.getSampleDataPath("..", sensorType, TestDataType.PARSED));
Assert.assertEquals(expectedMessages.size(), actualMessages.size());
for (int i = 0; i < actualMessages.size(); i++) {
- String expectedMessage = new String(expectedMessages.get(i));
- String actualMessage = new String(actualMessages.get(i));
+ String expectedMessage = new String(expectedMessages.get(i), StandardCharsets.UTF_8);
+ String actualMessage = new String(actualMessages.get(i), StandardCharsets.UTF_8);
try {
ValidationUtils.assertJsonEqual(expectedMessage, actualMessage);
} catch (Throwable t) {
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserQueryTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserQueryTest.java
index babb0e2..6321d90 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserQueryTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserQueryTest.java
@@ -20,6 +20,7 @@ package org.apache.metron.parsers.json;
import static org.hamcrest.CoreMatchers.equalTo;
import com.google.common.collect.ImmutableMap;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import org.adrianwalker.multilinestring.Multiline;
@@ -63,7 +64,7 @@ public class JSONMapParserQueryTest {
parser.configure(new HashMap<String, Object>() {{
put(JSONMapParser.JSONP_QUERY, "$.foo");
}});
- List<JSONObject> output = parser.parse(JSON_LIST.getBytes());
+ List<JSONObject> output = parser.parse(JSON_LIST.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(2, output.size());
JSONObject message = output.get(0);
// account for timestamp field in the size
@@ -99,7 +100,7 @@ public class JSONMapParserQueryTest {
put(JSONMapParser.JSONP_QUERY, "$.foo");
put(JSONMapParser.OVERRIDE_ORIGINAL_STRING, true);
}});
- List<JSONObject> output = parser.parse(JSON_LIST.getBytes());
+ List<JSONObject> output = parser.parse(JSON_LIST.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(2, output.size());
JSONObject message = output.get(0);
@@ -135,7 +136,7 @@ public class JSONMapParserQueryTest {
parser.configure(new HashMap<String, Object>() {{
put(JSONMapParser.JSONP_QUERY, "$$..$$SDSE$#$#.");
}});
- List<JSONObject> output = parser.parse(JSON_LIST.getBytes());
+ List<JSONObject> output = parser.parse(JSON_LIST.getBytes(StandardCharsets.UTF_8));
}
@@ -145,7 +146,7 @@ public class JSONMapParserQueryTest {
parser.configure(new HashMap<String, Object>() {{
put(JSONMapParser.JSONP_QUERY, "$.foo");
}});
- List<JSONObject> output = parser.parse(JSON_SINGLE.getBytes());
+ List<JSONObject> output = parser.parse(JSON_SINGLE.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(0, output.size());
}
@@ -171,7 +172,7 @@ public class JSONMapParserQueryTest {
parser.configure(new HashMap<String, Object>() {{
put(JSONMapParser.JSONP_QUERY, "$.foo");
}});
- List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes());
+ List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(output.size(), 2);
//don't forget the timestamp field!
@@ -193,7 +194,7 @@ public class JSONMapParserQueryTest {
.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ERROR.name(),
JSONMapParser.JSONP_QUERY, "$.foo"));
UnitTestHelper.setLog4jLevel(BasicParser.class, Level.FATAL);
- parser.parse(collectionHandlingJSON.getBytes());
+ parser.parse(collectionHandlingJSON.getBytes(StandardCharsets.UTF_8));
UnitTestHelper.setLog4jLevel(BasicParser.class, Level.ERROR);
}
@@ -204,7 +205,7 @@ public class JSONMapParserQueryTest {
parser.configure(ImmutableMap
.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ALLOW.name(),
JSONMapParser.JSONP_QUERY, "$.foo"));
- List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes());
+ List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(output.size(), 2);
Assert.assertEquals(output.get(0).size(), 2);
JSONObject message = output.get(0);
@@ -223,7 +224,7 @@ public class JSONMapParserQueryTest {
parser.configure(ImmutableMap
.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.UNFOLD.name(),
JSONMapParser.JSONP_QUERY, "$.foo"));
- List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes());
+ List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(output.size(), 2);
Assert.assertEquals(output.get(0).size(), 5);
JSONObject message = output.get(0);
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java
index 3cdc346..f10e930 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java
@@ -19,6 +19,7 @@ package org.apache.metron.parsers.json;
import com.google.common.collect.ImmutableMap;
import java.util.List;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.log4j.Level;
import org.apache.metron.parsers.BasicParser;
@@ -42,7 +43,7 @@ public class JSONMapParserTest {
@Test
public void testHappyPath() {
JSONMapParser parser = new JSONMapParser();
- List<JSONObject> output = parser.parse(happyPathJSON.getBytes());
+ List<JSONObject> output = parser.parse(happyPathJSON.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(output.size(), 1);
//don't forget the timestamp field!
Assert.assertEquals(output.get(0).size(), 4);
@@ -77,7 +78,7 @@ public class JSONMapParserTest {
@Test
public void testCollectionHandlingDrop() {
JSONMapParser parser = new JSONMapParser();
- List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes());
+ List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(output.size(), 1);
//don't forget the timestamp field!
Assert.assertEquals(output.get(0).size(), 1);
@@ -91,7 +92,7 @@ public class JSONMapParserTest {
JSONMapParser parser = new JSONMapParser();
parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ERROR.name()));
UnitTestHelper.setLog4jLevel(BasicParser.class, Level.FATAL);
- parser.parse(collectionHandlingJSON.getBytes());
+ parser.parse(collectionHandlingJSON.getBytes(StandardCharsets.UTF_8));
UnitTestHelper.setLog4jLevel(BasicParser.class, Level.ERROR);
}
@@ -100,7 +101,7 @@ public class JSONMapParserTest {
public void testCollectionHandlingAllow() {
JSONMapParser parser = new JSONMapParser();
parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ALLOW.name()));
- List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes());
+ List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(output.size(), 1);
//don't forget the timestamp field!
Assert.assertEquals(output.get(0).size(), 2);
@@ -113,7 +114,7 @@ public class JSONMapParserTest {
public void testCollectionHandlingUnfold() {
JSONMapParser parser = new JSONMapParser();
parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.UNFOLD.name()));
- List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes());
+ List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(output.size(), 1);
//don't forget the timestamp field!
Assert.assertEquals(output.get(0).size(), 5);
@@ -130,7 +131,8 @@ public class JSONMapParserTest {
public void testMixedCollectionHandlingUnfold() {
JSONMapParser parser = new JSONMapParser();
parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG,JSONMapParser.MapStrategy.UNFOLD.name()));
- List<JSONObject> output = parser.parse(mixCollectionHandlingJSON.getBytes());
+ List<JSONObject> output = parser.parse(mixCollectionHandlingJSON.getBytes(
+ StandardCharsets.UTF_8));
Assert.assertEquals(output.get(0).size(), 3);
JSONObject message = output.get(0);
Assert.assertEquals(message.get("collection.key"), "value");
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserWrappedQueryTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserWrappedQueryTest.java
index d5399d5..f3e5d98 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserWrappedQueryTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/json/JSONMapParserWrappedQueryTest.java
@@ -18,6 +18,7 @@
package org.apache.metron.parsers.json;
import com.google.common.collect.ImmutableMap;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.log4j.Level;
import org.apache.metron.parsers.BasicParser;
@@ -58,7 +59,7 @@ public class JSONMapParserWrappedQueryTest {
put(JSONMapParser.WRAP_ENTITY_NAME,"foo");
put(JSONMapParser.JSONP_QUERY, "$.foo");
}});
- List<JSONObject> output = parser.parse(JSON_LIST.getBytes());
+ List<JSONObject> output = parser.parse(JSON_LIST.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(output.size(), 2);
//don't forget the timestamp field!
Assert.assertEquals(output.get(0).size(), 4);
@@ -88,7 +89,7 @@ public class JSONMapParserWrappedQueryTest {
parser.configure(new HashMap<String, Object>() {{
put(JSONMapParser.JSONP_QUERY, "$$..$$SDSE$#$#.");
}});
- List<JSONObject> output = parser.parse(JSON_LIST.getBytes());
+ List<JSONObject> output = parser.parse(JSON_LIST.getBytes(StandardCharsets.UTF_8));
}
@@ -98,7 +99,7 @@ public class JSONMapParserWrappedQueryTest {
parser.configure(new HashMap<String, Object>() {{
put(JSONMapParser.JSONP_QUERY, "$.foo");
}});
- List<JSONObject> output = parser.parse(JSON_SINGLE.getBytes());
+ List<JSONObject> output = parser.parse(JSON_SINGLE.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(0, output.size());
}
@@ -124,7 +125,7 @@ public class JSONMapParserWrappedQueryTest {
parser.configure(new HashMap<String, Object>() {{
put(JSONMapParser.JSONP_QUERY, "$.foo");
}});
- List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes());
+ List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(output.size(), 2);
//don't forget the timestamp field!
@@ -146,7 +147,7 @@ public class JSONMapParserWrappedQueryTest {
.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ERROR.name(),
JSONMapParser.JSONP_QUERY, "$.foo"));
UnitTestHelper.setLog4jLevel(BasicParser.class, Level.FATAL);
- parser.parse(collectionHandlingJSON.getBytes());
+ parser.parse(collectionHandlingJSON.getBytes(StandardCharsets.UTF_8));
UnitTestHelper.setLog4jLevel(BasicParser.class, Level.ERROR);
}
@@ -157,7 +158,7 @@ public class JSONMapParserWrappedQueryTest {
parser.configure(ImmutableMap
.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ALLOW.name(),
JSONMapParser.JSONP_QUERY, "$.foo"));
- List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes());
+ List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(output.size(), 2);
Assert.assertEquals(output.get(0).size(), 2);
JSONObject message = output.get(0);
@@ -176,7 +177,7 @@ public class JSONMapParserWrappedQueryTest {
parser.configure(ImmutableMap
.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.UNFOLD.name(),
JSONMapParser.JSONP_QUERY, "$.foo"));
- List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes());
+ List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(output.size(), 2);
Assert.assertEquals(output.get(0).size(), 5);
JSONObject message = output.get(0);
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParserTest.java
index 5097ec0..079d813 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/regex/RegularExpressionsParserTest.java
@@ -14,6 +14,7 @@
*/
package org.apache.metron.parsers.regex;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
@@ -266,7 +267,8 @@ public class RegularExpressionsParserTest {
}
private JSONObject parse(String message) throws Exception {
- List<JSONObject> result = regularExpressionsParser.parse(message.getBytes());
+ List<JSONObject> result = regularExpressionsParser.parse(message.getBytes(
+ StandardCharsets.UTF_8));
if (result.size() > 0) {
return result.get(0);
}
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/syslog/Syslog3164ParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/syslog/Syslog3164ParserTest.java
index 6e8fb40..8bf20fc 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/syslog/Syslog3164ParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/syslog/Syslog3164ParserTest.java
@@ -19,6 +19,7 @@
package org.apache.metron.parsers.syslog;
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.parsers.interfaces.MessageParserResult;
import org.json.simple.JSONObject;
import org.junit.Assert;
@@ -142,7 +143,7 @@ public class Syslog3164ParserTest {
Syslog3164Parser parser = new Syslog3164Parser();
Map<String, Object> config = new HashMap<>();
parser.configure(config);
- parser.parseOptionalResult(line.getBytes());
+ parser.parseOptionalResult(line.getBytes(StandardCharsets.UTF_8));
}
@Test
@@ -157,7 +158,8 @@ public class Syslog3164ParserTest {
.append(SYSLOG_LINE_MISSING)
.append("\n")
.append(SYSLOG_LINE_ALL);
- Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(builder.toString().getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(builder.toString().getBytes(
+ StandardCharsets.UTF_8));
Assert.assertNotNull(resultOptional);
Assert.assertTrue(resultOptional.isPresent());
List<JSONObject> parsedList = resultOptional.get().getMessages();
@@ -179,7 +181,8 @@ public class Syslog3164ParserTest {
.append("BOOM!\n")
.append(SYSLOG_LINE_ALL)
.append("\nOHMY!");
- Optional<MessageParserResult<JSONObject>> output = parser.parseOptionalResult(builder.toString().getBytes());
+ Optional<MessageParserResult<JSONObject>> output = parser.parseOptionalResult(builder.toString().getBytes(
+ StandardCharsets.UTF_8));
Assert.assertTrue(output.isPresent());
Assert.assertEquals(3,output.get().getMessages().size());
Assert.assertEquals(3,output.get().getMessageThrowables().size());
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
index 461fe3f..8cc1c19 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
@@ -20,6 +20,7 @@ package org.apache.metron.parsers.syslog;
import com.github.palindromicity.syslog.NilPolicy;
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.parsers.interfaces.MessageParserResult;
import org.json.simple.JSONObject;
import org.junit.Assert;
@@ -141,7 +142,7 @@ public class Syslog5424ParserTest {
}
parser.configure(config);
- parser.parseOptionalResult(line.getBytes());
+ parser.parseOptionalResult(line.getBytes(StandardCharsets.UTF_8));
}
@Test
@@ -157,7 +158,8 @@ public class Syslog5424ParserTest {
.append(SYSLOG_LINE_MISSING)
.append("\n")
.append(SYSLOG_LINE_ALL);
- Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(builder.toString().getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(builder.toString().getBytes(
+ StandardCharsets.UTF_8));
Assert.assertNotNull(resultOptional);
Assert.assertTrue(resultOptional.isPresent());
List<JSONObject> parsedList = resultOptional.get().getMessages();
@@ -180,7 +182,8 @@ public class Syslog5424ParserTest {
.append("BOOM!\n")
.append(SYSLOG_LINE_ALL)
.append("\nOHMY!");
- Optional<MessageParserResult<JSONObject>> output = parser.parseOptionalResult(builder.toString().getBytes());
+ Optional<MessageParserResult<JSONObject>> output = parser.parseOptionalResult(builder.toString().getBytes(
+ StandardCharsets.UTF_8));
Assert.assertTrue(output.isPresent());
Assert.assertEquals(3,output.get().getMessages().size());
Assert.assertEquals(3,output.get().getMessageThrowables().size());
@@ -193,14 +196,15 @@ public class Syslog5424ParserTest {
String timeStampString = null;
config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.DASH.name());
parser.configure(config);
- Optional<MessageParserResult<JSONObject>> output = parser.parseOptionalResult(SYSLOG_LINE_MISSING_DATE.getBytes());
+ Optional<MessageParserResult<JSONObject>> output = parser.parseOptionalResult(SYSLOG_LINE_MISSING_DATE.getBytes(
+ StandardCharsets.UTF_8));
Assert.assertNotNull(output);
Assert.assertTrue(output.isPresent());
Assert.assertNotNull(output.get().getMessages().get(0).get("timestamp").toString());
config.clear();
config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.NULL.name());
parser.configure(config);
- output = parser.parseOptionalResult(SYSLOG_LINE_MISSING_DATE.getBytes());
+ output = parser.parseOptionalResult(SYSLOG_LINE_MISSING_DATE.getBytes(StandardCharsets.UTF_8));
Assert.assertNotNull(output);
Assert.assertTrue(output.isPresent());
timeStampString = output.get().getMessages().get(0).get("timestamp").toString();
@@ -209,7 +213,7 @@ public class Syslog5424ParserTest {
config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.OMIT.name());
parser.configure(config);
- output = parser.parseOptionalResult(SYSLOG_LINE_MISSING_DATE.getBytes());
+ output = parser.parseOptionalResult(SYSLOG_LINE_MISSING_DATE.getBytes(StandardCharsets.UTF_8));
Assert.assertNotNull(output);
Assert.assertTrue(output.isPresent());
}
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
index 9ae9541..2fd32a6 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
@@ -22,6 +22,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.time.Clock;
import java.time.ZoneId;
import java.util.ArrayList;
@@ -108,7 +109,7 @@ public class BasicAsaParser extends BasicParser {
private void addGrok(String key, String pattern) throws GrokException {
Grok grok = new Grok();
InputStream patternStream = this.getClass().getResourceAsStream("/patterns/asa");
- grok.addPatternFromReader(new InputStreamReader(patternStream));
+ grok.addPatternFromReader(new InputStreamReader(patternStream, StandardCharsets.UTF_8));
grok.compile("%{" + pattern + "}");
grokers.put(key, grok);
}
@@ -118,7 +119,7 @@ public class BasicAsaParser extends BasicParser {
syslogGrok = new Grok();
InputStream syslogStream = this.getClass().getResourceAsStream("/patterns/asa");
try {
- syslogGrok.addPatternFromReader(new InputStreamReader(syslogStream));
+ syslogGrok.addPatternFromReader(new InputStreamReader(syslogStream, StandardCharsets.UTF_8));
syslogGrok.compile(syslogPattern);
} catch (GrokException e) {
LOG.error("[Metron] Failed to load grok patterns from jar", e);
@@ -144,12 +145,7 @@ public class BasicAsaParser extends BasicParser {
List<JSONObject> messages = new ArrayList<>();
Map<String, Object> syslogJson = new HashMap<String, Object>();
- try {
- logLine = new String(rawMessage, "UTF-8");
- } catch (UnsupportedEncodingException e) {
- LOG.error("[Metron] Could not read raw message", e);
- throw new RuntimeException(e.getMessage(), e);
- }
+ logLine = new String(rawMessage, StandardCharsets.UTF_8);
try {
LOG.debug("[Metron] Started parsing raw message: {}", logLine);
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
index 9a3e36d..b60049f 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
@@ -19,6 +19,7 @@
package org.apache.metron.parsers.bro;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.ArrayList;
@@ -62,7 +63,7 @@ public class BasicBroParser extends BasicParser {
String rawMessage = null;
List<JSONObject> messages = new ArrayList<>();
try {
- rawMessage = new String(msg, "UTF-8");
+ rawMessage = new String(msg, StandardCharsets.UTF_8);
_LOG.trace("[Metron] Received message: {}", rawMessage);
JSONObject cleanedMessage = cleaner.clean(rawMessage);
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java
index a807f13..ea1eab7 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java
@@ -20,6 +20,7 @@ package org.apache.metron.parsers.cef;
import java.lang.invoke.MethodHandles;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.time.Clock;
import java.util.ArrayList;
import java.util.HashMap;
@@ -42,7 +43,6 @@ public class CEFParser extends BasicParser {
protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String HEADER_CAPTURE_PATTERN = "[^\\|]*";
private static final String EXTENSION_CAPTURE_PATTERN = "(?<!\\\\)=";
- private static final Charset UTF_8 = Charset.forName("UTF-8");
private Pattern p;
private static final Pattern patternExtensions = Pattern.compile(EXTENSION_CAPTURE_PATTERN);
@@ -144,7 +144,7 @@ public class CEFParser extends BasicParser {
public List<JSONObject> parse(byte[] rawMessage) {
List<JSONObject> messages = new ArrayList<>();
- String cefString = new String(rawMessage, UTF_8);
+ String cefString = new String(rawMessage, StandardCharsets.UTF_8);
Matcher matcher = p.matcher(cefString);
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
index 1cf9fb7..0b2c73e 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
@@ -94,7 +94,7 @@ public class BasicFireEyeParser extends BasicParser {
messages.add(toReturn);
return messages;
} catch (Exception e) {
- String message = "Unable to parse " + new String(rawMessage) + ": " + e.getMessage();
+ String message = "Unable to parse " + new String(rawMessage, StandardCharsets.UTF_8) + ": " + e.getMessage();
LOG.error(message, e);
throw new IllegalStateException(message, e);
}
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java
index 4e91585..a396eaf 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java
@@ -20,6 +20,7 @@
package org.apache.metron.parsers.ise;
import com.esotericsoftware.minlog.Log;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.parsers.BasicParser;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
@@ -55,7 +56,7 @@ public class BasicIseParser extends BasicParser {
List<JSONObject> messages = new ArrayList<>();
try {
- raw_message = new String(msg, "UTF-8");
+ raw_message = new String(msg, StandardCharsets.UTF_8);
_LOG.debug("Received message: {}", raw_message);
/*
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java
index 4845b4f..f23f5bc 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java
@@ -19,6 +19,8 @@
/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=false */
package org.apache.metron.parsers.ise;
+import java.nio.charset.StandardCharsets;
+
/**
* An implementation of interface CharStream, where the stream is assumed to
* contain only ASCII characters (with java-like unicode escape processing).
@@ -473,14 +475,14 @@ class JavaCharStream
public JavaCharStream(java.io.InputStream dstream, String encoding, int startline,
int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
{
- this(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+ this(encoding == null ? new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
}
/** Constructor. */
public JavaCharStream(java.io.InputStream dstream, int startline,
int startcolumn, int buffersize)
{
- this(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize);
+ this(new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8), startline, startcolumn, buffersize);
}
/** Constructor. */
@@ -513,14 +515,14 @@ class JavaCharStream
public void ReInit(java.io.InputStream dstream, String encoding, int startline,
int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
{
- ReInit(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+ ReInit(encoding == null ? new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
}
/** Reinitialise. */
public void ReInit(java.io.InputStream dstream, int startline,
int startcolumn, int buffersize)
{
- ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize);
+ ReInit(new java.io.InputStreamReader(dstream, StandardCharsets.UTF_8), startline, startcolumn, buffersize);
}
/** Reinitialise. */
public void ReInit(java.io.InputStream dstream, String encoding, int startline,
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java
index b12e98f..671833c 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java
@@ -19,6 +19,7 @@
package org.apache.metron.parsers.lancope;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
@@ -55,7 +56,7 @@ public class BasicLancopeParser extends BasicParser {
List<JSONObject> messages = new ArrayList<>();
try {
- String raw_message = new String(msg, "UTF-8");
+ String raw_message = new String(msg, StandardCharsets.UTF_8);
payload = (JSONObject) JSONValue.parse(raw_message);
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java
index ea09714..f5d96b2 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java
@@ -81,7 +81,6 @@ public class LEEFParser extends BasicParser {
protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String HEADER_CAPTURE_PATTERN = "[^\\|]*";
- private static final Charset UTF_8 = StandardCharsets.UTF_8;
private Pattern pattern;
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
index f10b976..783fac4 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.parsers.logstash;
+import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
@@ -48,7 +49,7 @@ public class BasicLogstashParser extends BasicParser {
* node, then transfered to the workers.
*/
JSONParser jsonParser = new JSONParser();
- String rawString = new String(raw_message, "UTF-8");
+ String rawString = new String(raw_message, StandardCharsets.UTF_8);
JSONObject rawJson = (JSONObject) jsonParser.parse(rawString);
// remove logstash meta fields
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
index c8e8b62..c0a6efe 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
@@ -20,6 +20,7 @@ package org.apache.metron.parsers.paloalto;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.parsers.BasicParser;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
@@ -177,7 +178,7 @@ public class BasicPaloAltoFirewallParser extends BasicParser {
List<JSONObject> messages = new ArrayList<>();
try {
- toParse = new String(msg, "UTF-8");
+ toParse = new String(msg, StandardCharsets.UTF_8);
_LOG.debug("Received message: {}", toParse);
parseMessage(toParse, outputMessage);
long timestamp = System.currentTimeMillis();
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
index 4dfe19a..fd9e23d 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
@@ -19,6 +19,7 @@ package org.apache.metron.parsers.snort;
import com.google.common.collect.Lists;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.time.ZoneId;
import java.time.ZonedDateTime;
@@ -139,7 +140,7 @@ public class BasicSnortParser extends BasicParser {
List<JSONObject> messages = new ArrayList<>();
try {
// snort alerts expected as csv records
- String csvMessage = new String(rawMessage, "UTF-8");
+ String csvMessage = new String(rawMessage, StandardCharsets.UTF_8);
Map<String, String> records = null;
try {
records = converter.toMap(csvMessage);
@@ -175,7 +176,8 @@ public class BasicSnortParser extends BasicParser {
jsonMessage.put("is_alert", "true");
messages.add(jsonMessage);
} catch (Exception e) {
- String message = "Unable to parse message: " + (rawMessage == null?"null" : new String(rawMessage));
+ String message = "Unable to parse message: " + (rawMessage == null?"null" : new String(rawMessage,
+ StandardCharsets.UTF_8));
_LOG.error(message, e);
throw new IllegalStateException(message, e);
}
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
index d7ead05..f8e09ff 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
@@ -18,6 +18,7 @@
package org.apache.metron.parsers.sourcefire;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -60,7 +61,7 @@ public class BasicSourcefireParser extends BasicParser {
List<JSONObject> messages = new ArrayList<>();
try {
- toParse = new String(msg, "UTF-8");
+ toParse = new String(msg, StandardCharsets.UTF_8);
_LOG.debug("Received message: {}", toParse);
String tmp = toParse.substring(toParse.lastIndexOf("{"));
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java
index 1981bb8..2264770 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java
@@ -18,6 +18,7 @@
package org.apache.metron.parsers;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.log4j.Level;
import org.apache.metron.common.Constants;
@@ -53,7 +54,7 @@ public class SnortParserTest {
public void testGoodMessage() {
BasicSnortParser parser = new BasicSnortParser();
parser.configure(new HashMap());
- Map out = parser.parse(goodMessage.getBytes()).get(0);
+ Map out = parser.parse(goodMessage.getBytes(StandardCharsets.UTF_8)).get(0);
Assert.assertEquals(out.get("msg"), "Consecutive TCP small segments, exceeding threshold");
Assert.assertEquals(out.get("sig_rev"), "1");
Assert.assertEquals(out.get("ip_dst_addr"), "10.0.2.15");
@@ -89,7 +90,7 @@ public class SnortParserTest {
BasicSnortParser parser = new BasicSnortParser();
parser.init();
UnitTestHelper.setLog4jLevel(BasicSnortParser.class, Level.FATAL);
- parser.parse("foo bar".getBytes());
+ parser.parse("foo bar".getBytes(StandardCharsets.UTF_8));
UnitTestHelper.setLog4jLevel(BasicSnortParser.class, Level.ERROR);
}
@@ -101,7 +102,7 @@ public class SnortParserTest {
TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of("America/New_York")));
BasicSnortParser parser = new BasicSnortParser();
parser.configure(new HashMap());
- Map out = parser.parse(goodMessage.getBytes()).get(0);
+ Map out = parser.parse(goodMessage.getBytes(StandardCharsets.UTF_8)).get(0);
Assert.assertEquals(out.get("timestamp"), 1453928464877L);
} finally {
// make sure we don't mess with other tests
@@ -122,7 +123,7 @@ public class SnortParserTest {
parserConfig.put("timeZone", "America/New_York");
BasicSnortParser parser = new BasicSnortParser();
parser.configure(parserConfig);
- Map result = parser.parse(dateFormattedMessage.getBytes()).get(0);
+ Map result = parser.parse(dateFormattedMessage.getBytes(StandardCharsets.UTF_8)).get(0);
assertThat("timestamp should match", result.get(Constants.Fields.TIMESTAMP.getName()), equalTo(1453928464877L));
}
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/asa/BasicAsaParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/asa/BasicAsaParserTest.java
index 12c39ca..1dd2e79 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/asa/BasicAsaParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/asa/BasicAsaParserTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.parsers.asa;
+import java.nio.charset.StandardCharsets;
import org.apache.log4j.Level;
import org.apache.metron.test.utils.UnitTestHelper;
import org.json.simple.JSONObject;
@@ -80,7 +81,7 @@ public class BasicAsaParserTest {
@Test
public void testCISCOFW106023() {
String rawMessage = "<164>Aug 05 2016 01:01:34: %ASA-4-106023: Deny tcp src Inside:10.30.9.121/54580 dst Outside:192.168.135.51/42028 by access-group \"Inside_access_in\" [0x962df600, 0x0]";
- JSONObject asaJson = asaParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject asaJson = asaParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
assertEquals(asaJson.get("original_string"), rawMessage);
assertTrue(asaJson.get("ip_src_addr").equals("10.30.9.121"));
assertTrue(asaJson.get("ip_dst_addr").equals("192.168.135.51"));
@@ -92,7 +93,7 @@ public class BasicAsaParserTest {
@Test
public void testCISCOFW106006() {
String rawMessage = "<162>Aug 05 2016 01:02:25: %ASA-2-106006: Deny inbound UDP from 10.25.177.164/63279 to 10.2.52.71/161 on interface Inside";
- JSONObject asaJson = asaParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject asaJson = asaParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
assertEquals(asaJson.get("original_string"), rawMessage);
assertTrue(asaJson.get("ip_src_addr").equals("10.25.177.164"));
assertTrue(asaJson.get("ip_dst_addr").equals("10.2.52.71"));
@@ -113,7 +114,7 @@ public class BasicAsaParserTest {
fixedClockParser.deviceClock = fixedClock;
fixedClockParser.init();
- JSONObject asaJson = fixedClockParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject asaJson = fixedClockParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
assertEquals(asaJson.get("original_string"), rawMessage);
assertTrue(asaJson.get("ip_src_addr").equals("10.22.8.110"));
assertTrue(asaJson.get("ip_dst_addr").equals("192.111.72.8"));
@@ -125,7 +126,7 @@ public class BasicAsaParserTest {
@Test
public void testNoPatternForTag() {
String rawMessage = "<165>Aug 16 2016 04:08:36: %ASA-5-713049: Group = 172.22.136.20, IP = 172.22.136.20, Security negotiation complete for LAN-to-LAN Group (172.22.136.20) Initiator, Inbound SPI = 0x891fb03f, Outbound SPI = 0xbe4b5d8d";
- JSONObject asaJson = asaParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject asaJson = asaParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
assertEquals(asaJson.get("original_string"), rawMessage);
assertTrue((long) asaJson.get("timestamp") == 1471320516000L);
}
@@ -133,7 +134,7 @@ public class BasicAsaParserTest {
@Test
public void testInvalidIpAddr() {
String rawMessage = "<164>Aug 05 2016 01:01:34: %ASA-4-106023: Deny tcp src Inside:10.30.9.121/54580 dst Outside:192.168.256.51/42028 by access-group \"Inside_access_in\" [0x962df600, 0x0]";
- JSONObject asaJson = asaParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject asaJson = asaParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
assertEquals(asaJson.get("original_string"), rawMessage);
assertTrue((long) asaJson.get("timestamp") == 1470358894000L);
assertNull(asaJson.get("ip_dst_addr"));
@@ -142,7 +143,7 @@ public class BasicAsaParserTest {
@Test
public void testIp6Addr() {
String rawMessage = "<174>Jan 05 2016 14:52:35 10.22.8.212 %ASA-6-302015: Built inbound UDP connection 76245506 for outside:2001:db8:85a3::8a2e:370:7334/49886 (10.22.8.110/49886) to inside:2001:0db8:85a3:0000:0000:8a2e:0370:7334/8612 (192.111.72.8/8612) (user.name)";
- JSONObject asaJson = asaParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject asaJson = asaParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
assertEquals(rawMessage, asaJson.get("original_string"));
assertEquals("2001:db8:85a3::8a2e:370:7334", asaJson.get("ip_src_addr"));
assertEquals("2001:0db8:85a3:0000:0000:8a2e:0370:7334", asaJson.get("ip_dst_addr"));
@@ -154,21 +155,21 @@ public class BasicAsaParserTest {
@Test
public void testSyslogIpHost() {
String rawMessage = "<174>Jan 5 14:52:35 10.22.8.212 %ASA-6-302015: Built inbound UDP connection 76245506 for outside:10.22.8.110/49886 (10.22.8.110/49886) to inside:192.111.72.8/8612 (192.111.72.8/8612) (user.name)";
- JSONObject asaJson = asaParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject asaJson = asaParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
assertEquals("10.22.8.212", asaJson.get("syslog_host"));
}
@Test
public void testSyslogHost() {
String rawMessage = "<174>Jan 5 14:52:35 hostname-2 %ASA-6-302015: Built inbound UDP connection 76245506 for outside:10.22.8.110/49886 (10.22.8.110/49886) to inside:192.111.72.8/8612 (192.111.72.8/8612) (user.name)";
- JSONObject asaJson = asaParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject asaJson = asaParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
assertEquals("hostname-2", asaJson.get("syslog_host"));
}
@Test
public void testSyslogHostAndProg() {
String rawMessage = "<174>Jan 5 14:52:35 hostname-2 progName-2 %ASA-6-302015: Built inbound UDP connection 76245506 for outside:10.22.8.110/49886 (10.22.8.110/49886) to inside:192.111.72.8/8612 (192.111.72.8/8612) (user.name)";
- JSONObject asaJson = asaParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject asaJson = asaParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
assertEquals("hostname-2", asaJson.get("syslog_host"));
assertEquals("progName-2", asaJson.get("syslog_prog"));
}
@@ -182,7 +183,7 @@ public class BasicAsaParserTest {
UnitTestHelper.setLog4jLevel(BasicAsaParser.class, Level.FATAL);
thrown.expect(RuntimeException.class);
thrown.expectMessage(startsWith("[Metron] Message '-- MARK --'"));
- JSONObject asaJson = asaParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject asaJson = asaParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
UnitTestHelper.setLog4jLevel(BasicAsaParser.class, Level.ERROR);
}
}
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
index 9d716e5..f5adbe4 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.parsers.bro;
+import java.nio.charset.StandardCharsets;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.log4j.Level;
@@ -95,7 +96,7 @@ public class BasicBroParserTest {
@Test
public void testUnwrappedBroMessage() throws ParseException {
JSONObject rawJson = (JSONObject)jsonParser.parse(unwrappedBroMessage);
- JSONObject broJson = broParser.parse(unwrappedBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(unwrappedBroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1449511228.474";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
@@ -149,7 +150,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(httpBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(httpBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(httpBroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1402307733.473";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1402307733473";
@@ -183,7 +184,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(rawMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedTimestamp = "1467657279000";
Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp);
String expectedBroTimestamp = "1467657279.0";
@@ -196,7 +197,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(rawMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedTimestamp = "1467657279000";
Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp);
String expectedBroTimestamp = "1467657279.0";
@@ -209,7 +210,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(rawMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedTimestamp = "1467657279100";
Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp);
String expectedBroTimestamp = "1467657279.1";
@@ -222,7 +223,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(rawMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(rawMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedTimestamp = "1467657279110";
Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp);
String expectedBroTimestamp = "1467657279.11";
@@ -263,7 +264,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(httpBroDecimalMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(httpBroDecimalMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(httpBroDecimalMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1457149494.166991";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1457149494166";
@@ -321,7 +322,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(dnsBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(dnsBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(dnsBroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1402308259.609";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1402308259609";
@@ -370,7 +371,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(filesBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(filesBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(filesBroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1425845251.334";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1425845251334";
@@ -420,7 +421,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(connBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(connBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(connBroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1166289883.163553";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1166289883163";
@@ -472,7 +473,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(dpdBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(dpdBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(dpdBroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1216704078.712276";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1216704078712";
@@ -520,7 +521,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(ftpBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(ftpBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(ftpBroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1166289883.164645";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1166289883164";
@@ -565,7 +566,8 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(knownCertsBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(knownCertsBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(knownCertsBroMessage.getBytes(
+ StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1216706999.896836";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1216706999896";
@@ -609,7 +611,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(smtpBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(smtpBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(smtpBroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1258568059.130219";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1258568059130";
@@ -662,7 +664,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(sslBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(sslBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(sslBroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1216706999.444925";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1216706999444";
@@ -711,7 +713,8 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(weirdBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(weirdBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(weirdBroMessage.getBytes(
+ StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1216706886.239896";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1216706886239";
@@ -761,7 +764,8 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(noticeBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(noticeBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(noticeBroMessage.getBytes(
+ StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1216706377.196728";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1216706377196";
@@ -812,7 +816,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(dhcpBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(dhcpBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(dhcpBroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1258567562.944638";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1258567562944";
@@ -862,7 +866,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(sshBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(sshBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(sshBroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1320435870.747967";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1320435870747";
@@ -911,7 +915,8 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(softwareBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(softwareBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(softwareBroMessage.getBytes(
+ StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1216707079.49066";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1216707079490";
@@ -951,7 +956,8 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(softwareBroMessage2);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(softwareBroMessage2.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(softwareBroMessage2.getBytes(
+ StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1216707079.518447";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1216707079518";
@@ -990,7 +996,8 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(radiusBroMessageFailed);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(radiusBroMessageFailed.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(radiusBroMessageFailed.getBytes(
+ StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1440447766.441298";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1440447766441";
@@ -1030,7 +1037,8 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(radiusBroMessageSuccess);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(radiusBroMessageSuccess.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(radiusBroMessageSuccess.getBytes(
+ StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1440447839.947956";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1440447839947";
@@ -1076,7 +1084,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(x509BroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(x509BroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(x509BroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1216706999.661483";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1216706999661";
@@ -1119,7 +1127,8 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(knownDevicesBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(knownDevicesBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(knownDevicesBroMessage.getBytes(
+ StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1258532046.693816";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1258532046693";
@@ -1163,7 +1172,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(rfbBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(rfbBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(rfbBroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1328634261.675248";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1328634261675";
@@ -1225,7 +1234,8 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(statsBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(statsBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(statsBroMessage.getBytes(
+ StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1440447766.440305";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1440447766440";
@@ -1277,7 +1287,8 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(captureLossBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(captureLossBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(captureLossBroMessage.getBytes(
+ StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1320435958.419451";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1320435958419";
@@ -1328,7 +1339,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(sipBroMessage);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(sipBroMessage.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(sipBroMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1216698441.346819";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1216698441346";
@@ -1391,7 +1402,7 @@ public class BasicBroParserTest {
Map rawMessageMap = (Map) jsonParser.parse(protocolKeyCleanedUp);
JSONObject rawJson = (JSONObject) rawMessageMap.get(rawMessageMap.keySet().iterator().next());
- JSONObject broJson = broParser.parse(protocolKeyCleanedUp.getBytes()).get(0);
+ JSONObject broJson = broParser.parse(protocolKeyCleanedUp.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedBroTimestamp = "1402307733.473";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
String expectedTimestamp = "1402307733473";
@@ -1405,11 +1416,11 @@ public class BasicBroParserTest {
@Test(expected=IllegalStateException.class)
public void testBadMessage() throws ParseException {
- broParser.parse("{ \"foo\" : \"bar\"}".getBytes());
+ broParser.parse("{ \"foo\" : \"bar\"}".getBytes(StandardCharsets.UTF_8));
}
@Test(expected=IllegalStateException.class)
public void testBadMessageNonJson() {
- broParser.parse("foo bar".getBytes());
+ broParser.parse("foo bar".getBytes(StandardCharsets.UTF_8));
}
}
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
index e6ca142..c3a569a 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
@@ -18,24 +18,6 @@
package org.apache.metron.parsers.cef;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.metron.common.Constants.Fields;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -44,10 +26,23 @@ import com.github.fge.jsonschema.core.report.ProcessingReport;
import com.github.fge.jsonschema.main.JsonSchemaFactory;
import com.github.fge.jsonschema.main.JsonValidator;
import com.google.common.io.Resources;
+import org.apache.metron.common.Constants.Fields;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
-public class CEFParserTest {
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
- private static final Charset UTF_8 = StandardCharsets.UTF_8;
+public class CEFParserTest {
private CEFParser parser;
@Before
@@ -178,29 +173,29 @@ public class CEFParserTest {
@Test
public void testCEFParserAdallom() throws Exception {
- runTest("adallom", Resources.readLines(Resources.getResource(getClass(), "adallom.cef"), UTF_8),
- Resources.toString(Resources.getResource(getClass(), "adallom.schema"), UTF_8));
+ runTest("adallom", Resources.readLines(Resources.getResource(getClass(), "adallom.cef"), StandardCharsets.UTF_8),
+ Resources.toString(Resources.getResource(getClass(), "adallom.schema"), StandardCharsets.UTF_8));
}
@Test
public void testCEFParserCyberArk() throws Exception {
- runTest("cyberark", Resources.readLines(Resources.getResource(getClass(), "cyberark.cef"), UTF_8),
- Resources.toString(Resources.getResource(getClass(), "cyberark.schema"), UTF_8),
- Resources.toString(Resources.getResource(getClass(), "cyberark.json"), UTF_8));
+ runTest("cyberark", Resources.readLines(Resources.getResource(getClass(), "cyberark.cef"), StandardCharsets.UTF_8),
+ Resources.toString(Resources.getResource(getClass(), "cyberark.schema"), StandardCharsets.UTF_8),
+ Resources.toString(Resources.getResource(getClass(), "cyberark.json"), StandardCharsets.UTF_8));
}
@Test
public void testCEFParserWAF() throws Exception {
URL waf_url = Resources.getResource(getClass(), "waf.cef");
- runTest("waf", Resources.readLines(waf_url, UTF_8),
- Resources.toString(Resources.getResource(getClass(), "waf.schema"), UTF_8));
+ runTest("waf", Resources.readLines(waf_url, StandardCharsets.UTF_8),
+ Resources.toString(Resources.getResource(getClass(), "waf.schema"), StandardCharsets.UTF_8));
}
@Test
public void testPaloAltoCEF() throws Exception {
URL palo_url = Resources.getResource(getClass(), "palo.cef");
- runTest("palo", Resources.readLines(palo_url, UTF_8),
- Resources.toString(Resources.getResource(getClass(), "palo.schema"), UTF_8));
+ runTest("palo", Resources.readLines(palo_url, StandardCharsets.UTF_8),
+ Resources.toString(Resources.getResource(getClass(), "palo.schema"), StandardCharsets.UTF_8));
}
private void runTest(String name, List<String> lines, String schema) throws Exception {
@@ -276,7 +271,7 @@ public class CEFParserTest {
}
private List<JSONObject> parse(String string) {
- List<JSONObject> parse = parser.parse(string.getBytes(UTF_8));
+ List<JSONObject> parse = parser.parse(string.getBytes(StandardCharsets.UTF_8));
Assert.assertNotNull(parse);
return parse;
}
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
index 7a5d2e6..bde7c0a 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.parsers.fireeye;
+import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.Map.Entry;
import java.time.Year;
@@ -43,7 +44,7 @@ public class BasicFireEyeParserTest extends AbstractParserConfigTest {
@Test
public void testParse() throws ParseException {
for (String inputString : inputStrings) {
- JSONObject parsed = parser.parse(inputString.getBytes()).get(0);
+ JSONObject parsed = parser.parse(inputString.getBytes(StandardCharsets.UTF_8)).get(0);
Assert.assertNotNull(parsed);
JSONParser parser = new JSONParser();
@@ -67,7 +68,7 @@ public class BasicFireEyeParserTest extends AbstractParserConfigTest {
@SuppressWarnings("rawtypes")
@Test
public void testTimestampParsing() throws ParseException {
- JSONObject parsed = parser.parse(fireeyeMessage.getBytes()).get(0);
+ JSONObject parsed = parser.parse(fireeyeMessage.getBytes(StandardCharsets.UTF_8)).get(0);
JSONParser parser = new JSONParser();
Map json = (Map) parser.parse(parsed.toJSONString());
long expectedTimestamp = ZonedDateTime.of(Year.now(ZoneOffset.UTC).getValue(), 3, 19, 5, 24, 39, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java
index 050a2d2..0a7ece4 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/ise/BasicIseParserTest.java
@@ -20,6 +20,7 @@ package org.apache.metron.parsers.ise;
import com.github.fge.jsonschema.core.exceptions.ProcessingException;
import java.io.IOException;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.util.Map;
import org.apache.metron.parsers.AbstractParserConfigTest;
import org.json.simple.JSONObject;
@@ -44,7 +45,7 @@ public class BasicIseParserTest extends AbstractParserConfigTest {
public void testParse()
throws org.json.simple.parser.ParseException, IOException, ProcessingException {
for (String inputString : inputStrings) {
- JSONObject parsed = parser.parse(inputString.getBytes()).get(0);
+ JSONObject parsed = parser.parse(inputString.getBytes(StandardCharsets.UTF_8)).get(0);
Assert.assertNotNull(parsed);
JSONParser parser = new JSONParser();
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java
index 50fdcd0..ab4613f 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/lancope/BasicLancopeParserTest.java
@@ -20,6 +20,7 @@ package org.apache.metron.parsers.lancope;
import com.github.fge.jsonschema.core.exceptions.ProcessingException;
import java.io.IOException;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.util.Map;
import org.apache.metron.parsers.AbstractParserConfigTest;
import org.json.simple.JSONObject;
@@ -44,7 +45,7 @@ public class BasicLancopeParserTest extends AbstractParserConfigTest {
@Test
public void testParse() throws ParseException, IOException, ProcessingException {
for (String inputString : inputStrings) {
- JSONObject parsed = parser.parse(inputString.getBytes()).get(0);
+ JSONObject parsed = parser.parse(inputString.getBytes(StandardCharsets.UTF_8)).get(0);
Assert.assertNotNull(parsed);
JSONParser parser = new JSONParser();
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/leef/LEEFParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/leef/LEEFParserTest.java
index 2db3224..f50fe37 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/leef/LEEFParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/leef/LEEFParserTest.java
@@ -17,12 +17,12 @@
*/
package org.apache.metron.parsers.leef;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.*;
-
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.fge.jackson.JsonLoader;
+import com.github.fge.jsonschema.core.report.ProcessingReport;
+import com.github.fge.jsonschema.main.JsonSchemaFactory;
+import com.github.fge.jsonschema.main.JsonValidator;
+import com.google.common.io.Resources;
import org.apache.metron.common.Constants.Fields;
import org.apache.metron.parsers.interfaces.MessageParserResult;
import org.json.simple.JSONObject;
@@ -31,18 +31,15 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.github.fge.jackson.JsonLoader;
-import com.github.fge.jsonschema.core.report.ProcessingReport;
-import com.github.fge.jsonschema.main.JsonSchemaFactory;
-import com.github.fge.jsonschema.main.JsonValidator;
-import com.google.common.io.Resources;
+import java.nio.charset.StandardCharsets;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
public class LEEFParserTest {
- private static final Charset UTF_8 = StandardCharsets.UTF_8;
private LEEFParser parser;
@Before
@@ -137,8 +134,8 @@ public class LEEFParserTest {
@Test
public void testLEEFParserSample() throws Exception {
- runTest("sample", Resources.readLines(Resources.getResource(getClass(), "sample.leef"), UTF_8),
- Resources.toString(Resources.getResource(getClass(), "sample.schema"), UTF_8));
+ runTest("sample", Resources.readLines(Resources.getResource(getClass(), "sample.leef"), StandardCharsets.UTF_8),
+ Resources.toString(Resources.getResource(getClass(), "sample.schema"), StandardCharsets.UTF_8));
}
private void runTest(String name, List<String> lines, String schema) throws Exception {
@@ -236,7 +233,7 @@ public class LEEFParserTest {
}
private List<JSONObject> parse(String string) {
- Optional<MessageParserResult<JSONObject>> parse = parser.parseOptionalResult(string.getBytes(UTF_8));
+ Optional<MessageParserResult<JSONObject>> parse = parser.parseOptionalResult(string.getBytes(StandardCharsets.UTF_8));
Assert.assertTrue(parse.isPresent());
return parse.get().getMessages();
}
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
index cc6191c..f49cab4 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
@@ -20,6 +20,7 @@ package org.apache.metron.parsers.paloalto;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.parsers.AbstractParserConfigTest;
import org.json.simple.JSONObject;
import org.json.simple.parser.ParseException;
@@ -40,7 +41,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
public void testParseSystem61() throws ParseException {
final String SYSTEM_61 = "1,2017/08/11 12:37:58,008900008659,SYSTEM,general,1,2017/08/11 11:37:58,vsys1,eventId_test,object_test,Futureuse1_test,futureuse2_test,management,high,Description_test,1354,0x0";
- JSONObject actual = parser.parse(SYSTEM_61.getBytes()).get(0);
+ JSONObject actual = parser.parse(SYSTEM_61.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
@@ -70,7 +71,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
public void testParseSystem80() throws ParseException {
final String SYSTEM_80 = "1,2017/08/11 12:37:58,008900008659,SYSTEM,general,1,2017/08/11 11:37:58,vsys1,eventId_test,object_test,Futureuse1_test,futureuse2_test,management,high,Description_test,1354,0x0,12,34,45,0,virSys1,dev-something200-01";
- JSONObject actual = parser.parse(SYSTEM_80.getBytes()).get(0);
+ JSONObject actual = parser.parse(SYSTEM_80.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
@@ -107,7 +108,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
public void testParseConfig61NoCustomFields() throws ParseException {
final String CONFIG_61_customFields = "1,2017/08/11 12:37:58,008900008659,CONFIG,0,1,2017/08/11 11:37:58,192.168.14.162,vsys1,edit,admin,Web,Succeeded, config shared log-settings config,1354,0x0";
- JSONObject actual = parser.parse(CONFIG_61_customFields.getBytes()).get(0);
+ JSONObject actual = parser.parse(CONFIG_61_customFields.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
@@ -140,7 +141,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
public void testParseConfig61CustomFields() throws ParseException {
final String CONFIG_61_noCustomFields = "1,2017/08/11 12:37:58,008900008659,CONFIG,0,1,2017/08/11 11:37:58,192.168.14.162,vsys1,edit,admin,Web,Succeeded, config shared log-settings config,1354,0x0,/FatherNode/KidNode/GrandsonNode1,/FatherNode/KidNode/GrandsonNode2";
- JSONObject actual = parser.parse(CONFIG_61_noCustomFields.getBytes()).get(0);
+ JSONObject actual = parser.parse(CONFIG_61_noCustomFields.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
@@ -175,7 +176,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
public void testParseConfig70And80NoCustomFields() throws ParseException {
final String CONFIG_70_80_noCustomFields = "1,2017/08/11 12:37:58,008900008659,CONFIG,0,1,2017/08/11 11:37:58,192.168.14.162,vsys1,edit,admin,Web,Succeeded, config shared log-settings config,1354,0x0,12,34,45,0,virSys1,dev-something200-01";
- JSONObject actual = parser.parse(CONFIG_70_80_noCustomFields.getBytes()).get(0);
+ JSONObject actual = parser.parse(CONFIG_70_80_noCustomFields.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
@@ -214,7 +215,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
public void testParseConfig70And80CustomFields() throws ParseException {
final String CONFIG_70_80_customFields = "1,2017/08/11 12:37:58,008900008659,CONFIG,0,1,2017/08/11 11:37:58,192.168.14.162,vsys1,edit,admin,Web,Succeeded,config shared log-settings config,/FatherNode/KidNode/GrandsonNode1,/FatherNode/KidNode/GrandsonNode2,1354,0x0,12,34,45,0,virSys1,dev-something200-01";
- JSONObject actual = parser.parse(CONFIG_70_80_customFields.getBytes()).get(0);
+ JSONObject actual = parser.parse(CONFIG_70_80_customFields.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
@@ -255,7 +256,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
@SuppressWarnings("unchecked")
@Test
public void testParseThreat60() throws ParseException {
- JSONObject actual = parser.parse(THREAT_60.getBytes()).get(0);
+ JSONObject actual = parser.parse(THREAT_60.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.Action, "reset-both");
@@ -310,7 +311,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
@SuppressWarnings("unchecked")
@Test
public void testParseTraffic60() throws ParseException {
- JSONObject actual = parser.parse(TRAFFIC_60.getBytes()).get(0);
+ JSONObject actual = parser.parse(TRAFFIC_60.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.Action, "allow");
@@ -365,7 +366,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
@SuppressWarnings("unchecked")
@Test
public void testParseThreat70() throws ParseException {
- JSONObject actual = parser.parse(THREAT_70.getBytes()).get(0);
+ JSONObject actual = parser.parse(THREAT_70.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.Action, "reset-both");
@@ -426,7 +427,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
@SuppressWarnings("unchecked")
@Test
public void testParseTraffic70() throws ParseException {
- JSONObject actual = parser.parse(TRAFFIC_70.getBytes()).get(0);
+ JSONObject actual = parser.parse(TRAFFIC_70.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.Action, "deny");
@@ -487,7 +488,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
@SuppressWarnings("unchecked")
@Test
public void testParseTraffic71() throws ParseException {
- JSONObject actual = parser.parse(TRAFFIC_71.getBytes()).get(0);
+ JSONObject actual = parser.parse(TRAFFIC_71.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.Action, "deny");
@@ -548,7 +549,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
@SuppressWarnings("unchecked")
@Test
public void testParseThreat71() throws ParseException {
- JSONObject actual = parser.parse(THREAT_71.getBytes()).get(0);
+ JSONObject actual = parser.parse(THREAT_71.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.Action, "alert");
@@ -608,7 +609,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
@SuppressWarnings("unchecked")
@Test
public void testParseThreat80() throws ParseException {
- JSONObject actual = parser.parse(THREAT_80.getBytes()).get(0);
+ JSONObject actual = parser.parse(THREAT_80.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.Action, "reset-server");
@@ -673,7 +674,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
@SuppressWarnings("unchecked")
@Test
public void testParseTraffic80() throws ParseException {
- JSONObject actual = parser.parse(TRAFFIC_80.getBytes()).get(0);
+ JSONObject actual = parser.parse(TRAFFIC_80.getBytes(StandardCharsets.UTF_8)).get(0);
JSONObject expected = new JSONObject();
expected.put(BasicPaloAltoFirewallParser.Action, "allow");
@@ -737,7 +738,8 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
@Test
public void testParseInvalidLogTypeMessage() throws ParseException {
final String unsupportedLogTypeMessage = "1,2017/08/11 12:37:58,008900008659,INVALIDlogType,0,1,2017/08/11 11:37:58,192.168.14.162,vsys1,edit,admin,Web,Succeeded, config shared log-settings config,1354,0x0";
- List<JSONObject> actual = parser.parse(unsupportedLogTypeMessage.getBytes());
+ List<JSONObject> actual = parser.parse(unsupportedLogTypeMessage.getBytes(
+ StandardCharsets.UTF_8));
assertNull(actual);
}
@@ -747,7 +749,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
public void testParseInvalidVersionMessage() throws ParseException {
final String invalidLengthMessage = "1,2017/08/11 12:37:58,008900008659,CONFIG,0,1,2017/08/11 11:37:58,192.168.14.162,vsys1,edit,admin,Web,Succeeded, config shared log-settings config";
- JSONObject actual = parser.parse(invalidLengthMessage.getBytes()).get(0);
+ JSONObject actual = parser.parse(invalidLengthMessage.getBytes(StandardCharsets.UTF_8)).get(0);
String expectedParserVersion = actual.get(BasicPaloAltoFirewallParser.ParserVersion).toString();
assertEquals(expectedParserVersion, "0");
}
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java
index dedd9db..578b93f 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParserTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.parsers.sourcefire;
+import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.metron.parsers.AbstractParserConfigTest;
@@ -40,8 +41,8 @@ public class BasicSourcefireParserTest extends AbstractParserConfigTest {
@Test
public void testParse() throws ParseException {
for (String inputString : inputStrings) {
- byte[] srcBytes = inputString.getBytes();
- JSONObject parsed = parser.parse(inputString.getBytes()).get(0);
+ byte[] srcBytes = inputString.getBytes(StandardCharsets.UTF_8);
+ JSONObject parsed = parser.parse(inputString.getBytes(StandardCharsets.UTF_8)).get(0);
Assert.assertNotNull(parsed);
JSONParser parser = new JSONParser();
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
index 81a22be..469fb73 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
@@ -20,6 +20,7 @@ package org.apache.metron.parsers.websphere;
import static org.junit.Assert.assertEquals;
+import java.nio.charset.StandardCharsets;
import java.time.*;
import java.util.HashMap;
import java.util.List;
@@ -54,7 +55,8 @@ public class GrokWebSphereParserTest {
parser.configure(parserConfig);
String testString = "<133>Apr 15 17:47:28 ABCXML1413 [rojOut][0x81000033][auth][notice] user(rick007): "
+ "[120.43.200.6]: User logged into 'cohlOut'.";
- Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes(
+ StandardCharsets.UTF_8));
Assert.assertNotNull(resultOptional);
Assert.assertTrue(resultOptional.isPresent());
List<JSONObject> result = resultOptional.get().getMessages();
@@ -83,7 +85,8 @@ public class GrokWebSphereParserTest {
parser.configure(parserConfig);
String testString = "<134>Apr 15 18:02:27 PHIXML3RWD [0x81000019][auth][info] [14.122.2.201]: "
+ "User 'hjpotter' logged out from 'default'.";
- Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes(
+ StandardCharsets.UTF_8));
Assert.assertNotNull(resultOptional);
Assert.assertTrue(resultOptional.isPresent());
List<JSONObject> result = resultOptional.get().getMessages();
@@ -111,7 +114,8 @@ public class GrokWebSphereParserTest {
parser.configure(parserConfig);
String testString = "<131>Apr 15 17:36:35 ROBXML3QRS [0x80800018][auth][error] rbm(RBM-Settings): "
+ "trans(3502888135)[request] gtid(3502888135): RBM: Resource access denied.";
- Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes(
+ StandardCharsets.UTF_8));
Assert.assertNotNull(resultOptional);
Assert.assertTrue(resultOptional.isPresent());
List<JSONObject> result = resultOptional.get().getMessages();
@@ -138,7 +142,8 @@ public class GrokWebSphereParserTest {
parser.configure(parserConfig);
String testString = "<134>Apr 15 17:17:34 SAGPXMLQA333 [0x8240001c][audit][info] trans(191): (admin:default:system:*): "
+ "ntp-service 'NTP Service' - Operational state down";
- Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes(
+ StandardCharsets.UTF_8));
Assert.assertNotNull(resultOptional);
Assert.assertTrue(resultOptional.isPresent());
List<JSONObject> result = resultOptional.get().getMessages();
@@ -164,7 +169,8 @@ public class GrokWebSphereParserTest {
parser.configure(parserConfig);
String testString = "<133>Apr 15 17:47:28 ABCXML1413 [rojOut][0x81000033][auth][notice] rick007): "
+ "[120.43.200. User logged into 'cohlOut'.";
- Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes(
+ StandardCharsets.UTF_8));
Assert.assertNotNull(resultOptional);
Assert.assertTrue(resultOptional.isPresent());
List<JSONObject> result = resultOptional.get().getMessages();
@@ -193,7 +199,8 @@ public class GrokWebSphereParserTest {
parser.configure(parserConfig);
String testString = "<134>Apr 15 18:02:27 PHIXML3RWD [0x81000019][auth][info] [14.122.2.201: "
+ "User 'hjpotter' logged out from 'default.";
- Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes(
+ StandardCharsets.UTF_8));
Assert.assertNotNull(resultOptional);
Assert.assertTrue(resultOptional.isPresent());
List<JSONObject> result = resultOptional.get().getMessages();
@@ -221,7 +228,8 @@ public class GrokWebSphereParserTest {
parser.configure(parserConfig);
String testString = "<131>Apr 15 17:36:35 ROBXML3QRS [0x80800018][auth][error] rbmRBM-Settings): "
+ "trans3502888135)[request] gtid3502888135) RBM: Resource access denied.";
- Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes(
+ StandardCharsets.UTF_8));
Assert.assertNotNull(resultOptional);
Assert.assertTrue(resultOptional.isPresent());
List<JSONObject> result = resultOptional.get().getMessages();
@@ -248,7 +256,8 @@ public class GrokWebSphereParserTest {
parser.configure(parserConfig);
String testString = "<134>Apr 15 17:17:34 SAGPXMLQA333 [0x8240001c][audit][info] trans 191) admindefaultsystem*): "
+ "ntp-service 'NTP Service' - Operational state down:";
- Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes(
+ StandardCharsets.UTF_8));
Assert.assertNotNull(resultOptional);
Assert.assertTrue(resultOptional.isPresent());
List<JSONObject> result = resultOptional.get().getMessages();
diff --git a/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
index 764a445..2f4d0b6 100644
--- a/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
@@ -17,39 +17,32 @@
*/
package org.apache.metron.parsers.integration.validation;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.verify;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
import org.apache.commons.lang.SerializationUtils;
import org.apache.metron.common.configuration.IndexingConfigurations;
import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.error.MetronError;
-import org.apache.metron.common.writer.BulkMessageWriter;
import org.apache.metron.common.writer.BulkMessage;
+import org.apache.metron.common.writer.BulkMessageWriter;
import org.apache.metron.common.writer.BulkWriterResponse;
import org.apache.metron.common.writer.MessageId;
import org.apache.metron.integration.ProcessorResult;
import org.apache.metron.parsers.bolt.ParserBolt;
import org.apache.metron.parsers.bolt.WriterHandler;
import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.*;
+
public class StormParserDriver extends ParserDriver {
private static final Logger LOG = LoggerFactory.getLogger(StormParserDriver.class);
@@ -67,7 +60,7 @@ public class StormParserDriver extends ParserDriver {
@Override
public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, List<BulkMessage<JSONObject>> messages) throws Exception {
- messages.forEach(bulkWriterMessage -> output.add(bulkWriterMessage.getMessage().toJSONString().getBytes()));
+ messages.forEach(bulkWriterMessage -> output.add(bulkWriterMessage.getMessage().toJSONString().getBytes(StandardCharsets.UTF_8)));
Set<MessageId> ids = messages.stream().map(BulkMessage::getId).collect(Collectors.toSet());
BulkWriterResponse bulkWriterResponse = new BulkWriterResponse();
bulkWriterResponse.addAllSuccesses(ids);
diff --git a/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
index 3e0f9ca..28b7cbb 100644
--- a/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
+++ b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
@@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import java.io.IOException;
import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -307,7 +308,7 @@ public class WriterBoltIntegrationTest extends BaseIntegrationTest {
message -> {
try {
return new JSONObject(
- JSONUtils.INSTANCE.load(new String(message), JSONUtils.MAP_SUPPLIER));
+ JSONUtils.INSTANCE.load(new String(message, StandardCharsets.UTF_8), JSONUtils.MAP_SUPPLIER));
} catch (Exception ex) {
throw new IllegalStateException(ex);
}
@@ -524,7 +525,7 @@ public class WriterBoltIntegrationTest extends BaseIntegrationTest {
for (byte[] b : kc.readMessages(topic)) {
try {
JSONObject m = new JSONObject(
- JSONUtils.INSTANCE.load(new String(b), JSONUtils.MAP_SUPPLIER));
+ JSONUtils.INSTANCE.load(new String(b, StandardCharsets.UTF_8), JSONUtils.MAP_SUPPLIER));
out.add(m);
} catch (IOException e) {
throw new IllegalStateException(e);
diff --git a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index ec06c52..3049782 100644
--- a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++ b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -30,6 +30,7 @@ import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@@ -337,16 +338,17 @@ public class PcapCliTest {
* @param type Fixed|Query
* @param optMsg Expected error message
*/
- public void assertCliError(String[] args, String type, String optMsg) {
+ public void assertCliError(String[] args, String type, String optMsg)
+ throws UnsupportedEncodingException {
PrintStream originalOutStream = System.out;
PrintStream originalErrOutStream = System.err;
try {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
- PrintStream outStream = new PrintStream(new BufferedOutputStream(bos));
+ PrintStream outStream = new PrintStream(new BufferedOutputStream(bos), false, StandardCharsets.UTF_8.name());
System.setOut(outStream);
ByteArrayOutputStream ebos = new ByteArrayOutputStream();
- PrintStream errOutStream = new PrintStream(new BufferedOutputStream(ebos));
+ PrintStream errOutStream = new PrintStream(new BufferedOutputStream(ebos), false, StandardCharsets.UTF_8.name());
System.setErr(errOutStream);
PcapCli cli = new PcapCli(jobRunner, clock -> "random_prefix");
diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/schema/SchemaTranslator.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/schema/SchemaTranslator.java
index e17adc6..5834afa 100644
--- a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/schema/SchemaTranslator.java
+++ b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/schema/SchemaTranslator.java
@@ -19,6 +19,7 @@ package org.apache.metron.solr.schema;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+import java.nio.charset.StandardCharsets;
import org.apache.metron.common.utils.JSONUtils;
import java.io.File;
@@ -187,7 +188,7 @@ public class SchemaTranslator {
String templateFile = argv[0];
String schemaFile = argv[1];
Map<String, Object> template = JSONUtils.INSTANCE.load(new File(templateFile), JSONUtils.MAP_SUPPLIER);
- try(PrintWriter pw = new PrintWriter(new File(schemaFile))) {
+ try(PrintWriter pw = new PrintWriter(new File(schemaFile), StandardCharsets.UTF_8.name())) {
translate(pw, template);
}
}
diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/converters/BinaryConverters.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/converters/BinaryConverters.java
index aa00672..2586116 100644
--- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/converters/BinaryConverters.java
+++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/converters/BinaryConverters.java
@@ -17,9 +17,11 @@
*/
package org.apache.metron.test.converters;
+import java.nio.charset.StandardCharsets;
+
public enum BinaryConverters implements IConverter {
DEFAULT(s -> {
- return s.getBytes();
+ return s.getBytes(StandardCharsets.UTF_8);
})
, FROM_HEX_STRING(new HexStringConverter());
IConverter _underlying;
diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/filereaders/FileReader.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/filereaders/FileReader.java
index 423e940..cf90d05 100644
--- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/filereaders/FileReader.java
+++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/filereaders/FileReader.java
@@ -19,6 +19,7 @@
package org.apache.metron.test.filereaders;
import java.io.*;
+import java.nio.charset.StandardCharsets;
import java.util.LinkedList;
import java.util.List;
@@ -39,7 +40,7 @@ public class FileReader {
.getResourceAsStream(filename);
}
DataInputStream in = new DataInputStream(stream);
- BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
String strLine;
while ((strLine = br.readLine()) != null)
{
diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java
index 45f63ae..0fdf9f0 100644
--- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java
+++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java
@@ -18,6 +18,7 @@
package org.apache.metron.test.spouts;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
@@ -114,7 +115,7 @@ public class GenericInternalTestSpout extends BaseRichSpout {
if (_converter != null) {
value = _converter.convert(jsons.get(cnt));
} else {
- value = jsons.get(cnt).getBytes();
+ value = jsons.get(cnt).getBytes(StandardCharsets.UTF_8);
}
_collector.emit(new Values(value));
}
diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/KafkaLoader.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/KafkaLoader.java
index 294f8fd..6812337 100644
--- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/KafkaLoader.java
+++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/KafkaLoader.java
@@ -17,6 +17,9 @@
*/
package org.apache.metron.test.utils;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
@@ -58,7 +61,7 @@ public class KafkaLoader {
kafkaProducer = new KafkaProducer<>(producerConfig);
try {
while (iterations == -1 || iterations-- > 0) {
- BufferedReader reader = new BufferedReader(new FileReader(samplePath));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(samplePath), StandardCharsets.UTF_8));
String line;
while((line = reader.readLine()) != null) {
kafkaProducer.send(new ProducerRecord<String, String>(topic, line));
diff --git a/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java b/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
index 9d4193f..72c7ea0 100644
--- a/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
+++ b/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
@@ -21,6 +21,7 @@ package org.apache.metron.writer.hdfs;
import java.io.FileOutputStream;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
@@ -69,7 +70,7 @@ public class SourceHandler {
protected void handle(JSONObject message, String sensor, WriterConfiguration config, SyncPolicyCreator syncPolicyCreator) throws IOException {
- byte[] bytes = (message.toJSONString() + "\n").getBytes();
+ byte[] bytes = (message.toJSONString() + "\n").getBytes(StandardCharsets.UTF_8);
synchronized (this.writeLock) {
try {
out.write(bytes);
diff --git a/metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/bolt/BulkMessageWriterBoltTest.java b/metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/bolt/BulkMessageWriterBoltTest.java
index 3caed2c..2256f26 100644
--- a/metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/bolt/BulkMessageWriterBoltTest.java
+++ b/metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/bolt/BulkMessageWriterBoltTest.java
@@ -33,6 +33,7 @@ import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
import java.io.FileInputStream;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -320,7 +321,7 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
bolt.prepare(stormConf, topologyContext, outputCollector, clock);
// execute a tuple that contains an invalid message
- byte[] invalidJSON = "this is not valid JSON".getBytes();
+ byte[] invalidJSON = "this is not valid JSON".getBytes(StandardCharsets.UTF_8);
when(tuple.getBinary(0)).thenReturn(invalidJSON);
bolt.execute(tuple);
diff --git a/metron-platform/metron-zookeeper/src/main/java/org/apache/metron/zookeeper/SimpleEventListener.java b/metron-platform/metron-zookeeper/src/main/java/org/apache/metron/zookeeper/SimpleEventListener.java
index 1078f1e..edc416c 100644
--- a/metron-platform/metron-zookeeper/src/main/java/org/apache/metron/zookeeper/SimpleEventListener.java
+++ b/metron-platform/metron-zookeeper/src/main/java/org/apache/metron/zookeeper/SimpleEventListener.java
@@ -19,6 +19,7 @@ package org.apache.metron.zookeeper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
+import java.nio.charset.StandardCharsets;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
import org.apache.curator.framework.recipes.cache.TreeCacheListener;
@@ -111,7 +112,8 @@ public class SimpleEventListener implements TreeCacheListener {
path = event.getData().getPath();
data = event.getData().getData();
}
- LOG.debug("Type: {}, Path: {}, Data: {}", event.getType(), (path == null?"":path) , (data == null?"":new String(data)));
+ LOG.debug("Type: {}, Path: {}, Data: {}", event.getType(), (path == null?"":path) , (data == null?"":new String(data,
+ StandardCharsets.UTF_8)));
List<Callback> callback = callbacks.get(event.getType());
if(callback != null) {
for(Callback cb : callback) {
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java
index b8f0997..29262b7 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java
@@ -18,6 +18,7 @@
package org.apache.metron.stellar.common.shell.cli;
import com.google.common.collect.Iterables;
+import java.nio.charset.StandardCharsets;
import org.jboss.aesh.complete.CompleteOperation;
import org.jboss.aesh.console.AeshContext;
import org.jboss.aesh.console.ConsoleOperation;
@@ -51,8 +52,8 @@ public class StellarShellTest {
err = new ByteArrayOutputStream();
// setup streams so that we can capture stdout
- System.setOut(new PrintStream(out));
- System.setErr(new PrintStream(err));
+ System.setOut(new PrintStream(out, false, StandardCharsets.UTF_8.name()));
+ System.setErr(new PrintStream(err, false, StandardCharsets.UTF_8.name()));
String[] args = new String[0];
stellarShell = new StellarShell(args);
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
index a746321..64477cf 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
@@ -335,7 +335,9 @@ public class RestFunctionsTest {
HttpEntity httpEntity = mock(HttpEntity.class);
// return successfully parsed response
- when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream("{\"get\":\"success\"}".getBytes()));
+ when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream("{\"get\":\"success\"}".getBytes(
+ StandardCharsets.UTF_8
+ )));
Optional<Object> actual = RestFunctions.parseResponse(restConfig, httpGet, httpEntity);
assertTrue(actual.isPresent());
assertEquals("success", ((Map<String, Object>) actual.get()).get("get"));
@@ -368,7 +370,7 @@ public class RestFunctionsTest {
HttpEntity httpEntity = mock(HttpEntity.class);
// return empty on empty input stream
- when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream("".getBytes()));
+ when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)));
assertEquals(Optional.empty(), RestFunctions.parseResponse(restConfig, httpGet, httpEntity));
}
@@ -384,7 +386,8 @@ public class RestFunctionsTest {
restConfig.put(VERIFY_CONTENT_LENGTH, true);
when(httpGet.getURI()).thenReturn(new URI("uri"));
- when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream("{\"get\":\"success\"}".getBytes()));
+ when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream("{\"get\":\"success\"}".getBytes(
+ StandardCharsets.UTF_8)));
when(httpEntity.getContentLength()).thenReturn(-1L);
RestFunctions.parseResponse(restConfig, httpGet, httpEntity);
}
diff --git a/pom.xml b/pom.xml
index edc2a03..6450331 100644
--- a/pom.xml
+++ b/pom.xml
@@ -255,8 +255,11 @@
<compilerId>javac-with-errorprone</compilerId>
<forceJavacCompilerUse>true</forceJavacCompilerUse>
<source>${global_java_version}</source>
- <compilerArgument>-Xlint:unchecked</compilerArgument>
<target>${global_java_version}</target>
+ <compilerArgs>
+ <arg>-Xlint:unchecked</arg>
+ <arg>-Xep:DefaultCharset:ERROR</arg>
+ </compilerArgs>
<showWarnings>true</showWarnings>
</configuration>
<dependencies>