You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@creadur.apache.org by cl...@apache.org on 2026/09/19 15:56:16 UTC
(creadur-rat) branch master updated: RAT-532: Bump tika.version from 3.3.2 to 4.0.0 and migrate charset detection (#714)
This is an automated email from the ASF dual-hosted git repository.
Claudenw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/creadur-rat.git
The following commit(s) were added to refs/heads/master by this push:
new 7766f240e RAT-532: Bump tika.version from 3.3.2 to 4.0.0 and migrate charset detection (#714)
7766f240e is described below
commit 7766f240ebc27167b6d84aa50494595715f1d4dc
Author: dependabot[bot] <49...@users.noreply.github.com>
AuthorDate: Sat Sep 19 17:56:10 2026 +0200
RAT-532: Bump tika.version from 3.3.2 to 4.0.0 and migrate charset detection (#714)
* Bump tika.version from 3.3.2 to 4.0.0
Bumps `tika.version` from 3.3.2 to 4.0.0.
Updates `org.apache.tika:tika-core` from 3.3.2 to 4.0.0
- [Changelog](https://github.com/apache/tika/blob/main/CHANGES.txt)
- [Commits](https://github.com/apache/tika/compare/3.3.2...4.0.0)
Updates `org.apache.tika:tika-parser-text-module` from 3.3.2 to 4.0.0
---
updated-dependencies:
- dependency-name: org.apache.tika:tika-core
dependency-version: 4.0.0
dependency-type: direct:production
update-type: version-update:semver-major
- dependency-name: org.apache.tika:tika-parser-text-module
dependency-version: 4.0.0
dependency-type: direct:production
update-type: version-update:semver-major
...
Signed-off-by: dependabot[bot] <su...@github.com>
* RAT-532: Upgrade to Tika 4.0.0
* Upgrade to Tika v4.0.0
* Migrate to new charset detection logic of Tika 4.
* It returns the best possible guess as encoding.
* In contrast to v3.x windows-1252 is detected instead of UTF-8/ISO-8859-1,
thus tests had to be changed as well for all UIs.
* Keep the performance optimisation (read only 256 bytes) introduced via RAT-494
* Log as warning if no encoding found
* Add test for character detection for empty contents and 'invalid data'
* Raise test coverage with unit tests
---------
Signed-off-by: dependabot[bot] <su...@github.com>
Co-authored-by: dependabot[bot] <49...@users.noreply.github.com>
Co-authored-by: P. Ottlinger <po...@apache.org>
Co-authored-by: P. Ottlinger <ot...@users.noreply.github.com>
---
.../it/resources/ReportTest/RAT_81/verify.groovy | 5 +-
.../org/apache/rat/analysis/TikaProcessor.java | 53 +++++++++++++++-------
.../src/main/java/org/apache/rat/api/Document.java | 24 ++++++----
.../org/apache/rat/ReporterOptionsProvider.java | 4 +-
.../src/test/java/org/apache/rat/ReporterTest.java | 22 ++++-----
.../org/apache/rat/analysis/TikaProcessorTest.java | 37 +++++++++++----
.../org/apache/rat/document/DocumentNameTest.java | 3 +-
.../org/apache/rat/document/FileDocumentTest.java | 25 ++++++++++
apache-rat-plugin/src/it/it1/verify.groovy | 2 +-
.../java/org/apache/rat/mp/RatCheckMojoTest.java | 4 +-
.../resources/antunit/report-normal-operation.xml | 2 +-
pom.xml | 2 +-
src/changes/changes.xml | 3 ++
13 files changed, 130 insertions(+), 56 deletions(-)
diff --git a/apache-rat-core/src/it/resources/ReportTest/RAT_81/verify.groovy b/apache-rat-core/src/it/resources/ReportTest/RAT_81/verify.groovy
index c1a5de0c4..a7981a11c 100644
--- a/apache-rat-core/src/it/resources/ReportTest/RAT_81/verify.groovy
+++ b/apache-rat-core/src/it/resources/ReportTest/RAT_81/verify.groovy
@@ -32,7 +32,8 @@ NodeList nodeList = XmlUtils.getNodeList(doc, xPath, "/rat-report/resource[@name
assertEquals(1, nodeList.getLength())
node = nodeList.item(0)
attributes = node.getAttributes()
-assertEquals("IBM500", attributes.getNamedItem("encoding").getNodeValue())
+// pre-Tika4: recognized as IBM500 instead of IBM1047
+assertEquals("IBM1047", attributes.getNamedItem("encoding").getNodeValue())
assertEquals("text/plain", attributes.getNamedItem("mediaType").getNodeValue())
assertEquals("STANDARD", attributes.getNamedItem("type").getNodeValue())
nodeList = XmlUtils.getNodeList(node, xPath, "license")
@@ -45,7 +46,7 @@ nodeList = XmlUtils.getNodeList(doc, xPath, "/rat-report/resource[@name='/UTF8.t
assertEquals(1, nodeList.getLength())
node = nodeList.item(0)
attributes = node.getAttributes()
-assertEquals("ISO-8859-1", attributes.getNamedItem("encoding").getNodeValue())
+assertEquals("windows-1252", attributes.getNamedItem("encoding").getNodeValue())
assertEquals("text/plain", attributes.getNamedItem("mediaType").getNodeValue())
assertEquals("STANDARD", attributes.getNamedItem("type").getNodeValue())
nodeList = XmlUtils.getNodeList(node, xPath, "license")
diff --git a/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java b/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java
index 86e464ad1..8534eac2c 100644
--- a/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java
+++ b/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java
@@ -24,6 +24,7 @@ import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.UnsupportedCharsetException;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.rat.api.Document;
@@ -32,19 +33,29 @@ import org.apache.rat.document.RatDocumentAnalysisException;
import org.apache.rat.document.guesser.NoteGuesser;
import org.apache.rat.utils.DefaultLog;
import org.apache.tika.Tika;
+import org.apache.tika.detect.DefaultEncodingDetector;
+import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.detect.EncodingResult;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
-import org.apache.tika.parser.txt.CharsetDetector;
-import org.apache.tika.parser.txt.CharsetMatch;
+import org.apache.tika.parser.ParseContext;
/**
* A wrapping around the Tika processor.
*/
public final class TikaProcessor {
- /** The Tika parser */
+ /** The Tika parser. */
private static final Tika TIKA = new Tika();
+
+ /** The Tika encoding detector. */
+ private static final EncodingDetector ENCODING_DETECTOR = new DefaultEncodingDetector();
+
+ /** Due to performance reasons we do not read the whole file for charset detection (RAT-494). */
+ private static final int BYTES_FOR_CHARSET_DETECTION = 256;
+
/** A map of mime type string to non-BINARY types.
* "text" types are already handled somewhere else
* BINARY unless listed here
@@ -165,21 +176,31 @@ public final class TikaProcessor {
* @throws IOException on IO error.
* @throws UnsupportedCharsetException on unsupported charset.
*/
- private static Charset detectCharset(final InputStream stream, final DocumentName documentName) throws IOException, UnsupportedCharsetException {
- final int bytesForCharsetDetection = 256;
- CharsetDetector encodingDetector = new CharsetDetector(bytesForCharsetDetection);
- encodingDetector.setText(stream);
- CharsetMatch charsetMatch = encodingDetector.detect();
- if (charsetMatch != null) {
- try {
- return Charset.forName(charsetMatch.getName());
- } catch (UnsupportedCharsetException e) {
- DefaultLog.getInstance().warn(String.format("Unsupported character set '%s' in file '%s'",
- charsetMatch.getName(), documentName));
- throw e;
+ static Charset detectCharset(final InputStream stream, final DocumentName documentName) throws IOException, UnsupportedCharsetException {
+ stream.mark(BYTES_FOR_CHARSET_DETECTION);
+ try {
+ byte[] sample = stream.readNBytes(BYTES_FOR_CHARSET_DETECTION);
+ if (sample.length == 0) {
+ DefaultLog.getInstance().debug(String.format("No contents in file '%s'", documentName));
+ return null;
+ }
+
+ Metadata metadata = new Metadata();
+ ParseContext parseContext = new ParseContext();
+
+ try (TikaInputStream tis = TikaInputStream.get(sample, metadata)) {
+ List<EncodingResult> results = ENCODING_DETECTOR.detect(tis, metadata, parseContext);
+
+ if (results.isEmpty()) {
+ DefaultLog.getInstance().warn(String.format("No encoding found for file '%s'", documentName));
+ return null;
+ }
+
+ return results.get(0).getCharset();
}
+ } finally {
+ stream.reset();
}
- return null;
}
/**
diff --git a/apache-rat-core/src/main/java/org/apache/rat/api/Document.java b/apache-rat-core/src/main/java/org/apache/rat/api/Document.java
index c0d7efbd6..d3c0d5e3d 100644
--- a/apache-rat-core/src/main/java/org/apache/rat/api/Document.java
+++ b/apache-rat-core/src/main/java/org/apache/rat/api/Document.java
@@ -20,13 +20,14 @@ package org.apache.rat.api;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.io.Reader;
+import java.nio.charset.Charset;
import java.util.SortedSet;
import org.apache.rat.analysis.TikaProcessor;
import org.apache.rat.document.DocumentName;
import org.apache.rat.document.DocumentNameMatcher;
-import org.apache.tika.parser.txt.CharsetDetector;
/**
* The representation of a document being scanned.
@@ -104,19 +105,24 @@ public abstract class Document implements Comparable<Document> {
}
/**
- * Reads the contents of this document.
+ * Reads the contents of this document and
+ * relies on the charset detection of the underlying Tika processor.
+ *
* @return <code>Reader</code> not null
* @throws IOException if this document cannot be read.
*/
public Reader reader() throws IOException {
- final int bytesForCharsetDetection = 256;
- CharsetDetector charsetDetector = new CharsetDetector(bytesForCharsetDetection);
- // RAT-494: Tika's CharsetDetector.getReader() may return null if the read can not be constructed due to I/O or encoding errors
- Reader result = charsetDetector.getReader(TikaProcessor.markSupportedInputStream(inputStream()), getMetaData().getCharset().name());
- if (result == null) {
- throw new IOException(String.format("Can not read document `%s`", getName()));
+ final Charset charset = getMetaData().getCharset();
+ if (charset == null) {
+ throw new IOException(
+ String.format(
+ "No charset detected for document `%s`",
+ getName()));
}
- return result;
+
+ return new InputStreamReader(
+ TikaProcessor.markSupportedInputStream(inputStream()),
+ charset);
}
/**
diff --git a/apache-rat-core/src/test/java/org/apache/rat/ReporterOptionsProvider.java b/apache-rat-core/src/test/java/org/apache/rat/ReporterOptionsProvider.java
index d068c089f..56ad9ba7e 100644
--- a/apache-rat-core/src/test/java/org/apache/rat/ReporterOptionsProvider.java
+++ b/apache-rat-core/src/test/java/org/apache/rat/ReporterOptionsProvider.java
@@ -866,7 +866,7 @@ class ReporterOptionsProvider extends AbstractOptionsProvider implements Argumen
TextUtils.assertContainsExactly(1, "?????: 1 ", actualText);
break;
case XML:
- TextUtils.assertContainsExactly(1, "<resource encoding=\"ISO-8859-1\" mediaType=\"text/plain\" name=\"/stylesheet\" type=\"STANDARD\">", actualText);
+ TextUtils.assertContainsExactly(1, "<resource encoding=\"windows-1252\" mediaType=\"text/plain\" name=\"/stylesheet\" type=\"STANDARD\">", actualText);
break;
case UNAPPROVED_LICENSES:
TextUtils.assertContainsExactly(1, "Files with unapproved licenses:" + System.lineSeparator() + " /stylesheet", actualText);
@@ -928,7 +928,7 @@ class ReporterOptionsProvider extends AbstractOptionsProvider implements Argumen
assertThat(output.getStatistic().getCounter(ClaimStatistic.Counter.UNAPPROVED)).isEqualTo(1);
output.format(config);
String actualText = baos.toString(StandardCharsets.UTF_8);
- TextUtils.assertContainsExactly(1, "<resource encoding=\"ISO-8859-1\" mediaType=\"text/plain\" name=\"/stylesheet\" type=\"STANDARD\">", actualText);
+ TextUtils.assertContainsExactly(1, "<resource encoding=\"windows-1252\" mediaType=\"text/plain\" name=\"/stylesheet\" type=\"STANDARD\">", actualText);
try (InputStream expected = StyleSheets.getStyleSheet("xml").ioSupplier().get();
InputStream actual = config.getStyleSheet().get()) {
diff --git a/apache-rat-core/src/test/java/org/apache/rat/ReporterTest.java b/apache-rat-core/src/test/java/org/apache/rat/ReporterTest.java
index e49f32b88..4dc54ac7b 100644
--- a/apache-rat-core/src/test/java/org/apache/rat/ReporterTest.java
+++ b/apache-rat-core/src/test/java/org/apache/rat/ReporterTest.java
@@ -244,30 +244,30 @@ public class ReporterTest {
Map<String, Map<String, String>> expected = new HashMap<>();
expected.put("/.hiddenDirectory", mapOf("isDirectory", "true", "mediaType", "application/octet-stream",
"type", "IGNORED"));
- expected.put("/ILoggerFactory.java", mapOf("encoding", "ISO-8859-1", "mediaType", "text/x-java-source",
+ expected.put("/ILoggerFactory.java", mapOf("encoding", "windows-1252", "mediaType", "text/x-java-source",
"type", "STANDARD"));
expected.put("/Image.png", mapOf("mediaType", "image/png", "type", "BINARY"));
- expected.put("/LICENSE", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain", "type", "NOTICE"));
- expected.put("/NOTICE", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain", "type", "NOTICE"));
- expected.put("/Source.java", mapOf("encoding", "ISO-8859-1", "mediaType", "text/x-java-source",
+ expected.put("/LICENSE", mapOf("encoding", "windows-1252", "mediaType", "text/plain", "type", "NOTICE"));
+ expected.put("/NOTICE", mapOf("encoding", "windows-1252", "mediaType", "text/plain", "type", "NOTICE"));
+ expected.put("/Source.java", mapOf("encoding", "windows-1252", "mediaType", "text/x-java-source",
"type", "STANDARD"));
- expected.put("/Text.txt", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain",
+ expected.put("/Text.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain",
"type", "STANDARD"));
- expected.put("/TextHttps.txt", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain",
+ expected.put("/TextHttps.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain",
"type", "STANDARD"));
- expected.put("/Xml.xml", mapOf("encoding", "ISO-8859-1", "mediaType", "application/xml",
+ expected.put("/Xml.xml", mapOf("encoding", "windows-1252", "mediaType", "application/xml",
"type", "STANDARD"));
- expected.put("/buildr.rb", mapOf("encoding", "ISO-8859-1", "mediaType", "text/x-ruby",
+ expected.put("/buildr.rb", mapOf("encoding", "windows-1252", "mediaType", "text/x-ruby",
"type", "STANDARD"));
expected.put("/dummy.jar", mapOf("mediaType", "application/java-archive",
"type", "ARCHIVE"));
- expected.put("/generated.txt", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain",
+ expected.put("/generated.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain",
"type", "IGNORED"));
expected.put("/plain.json", mapOf("mediaType", "application/json",
"type", "BINARY"));
- expected.put("/sub/Empty.txt", mapOf("encoding", "UTF-8", "mediaType", "text/plain",
+ expected.put("/sub/Empty.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain",
"type", "STANDARD"));
- expected.put("/tri.txt", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain",
+ expected.put("/tri.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain",
"type", "STANDARD"));
File output = testPath.resolve(".rat/testXMLOutput").toFile();
diff --git a/apache-rat-core/src/test/java/org/apache/rat/analysis/TikaProcessorTest.java b/apache-rat-core/src/test/java/org/apache/rat/analysis/TikaProcessorTest.java
index 925419412..4aef2126d 100644
--- a/apache-rat-core/src/test/java/org/apache/rat/analysis/TikaProcessorTest.java
+++ b/apache-rat-core/src/test/java/org/apache/rat/analysis/TikaProcessorTest.java
@@ -27,11 +27,13 @@ import org.apache.rat.test.utils.Resources;
import org.apache.rat.document.DocumentName;
import org.junit.jupiter.api.Test;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.nio.charset.Charset;
import java.nio.charset.MalformedInputException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
@@ -40,6 +42,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.SortedSet;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -52,7 +55,7 @@ public class TikaProcessorTest {
* @see <a href="https://issues.apache.org/jira/browse/RAT-81">RAT-81</a>
*/
@Test
- public void RAT81() {
+ void RAT81() {
// create a document that throws a MalformedInputException
Document doc = mkDocument(new InputStream() {
@Override
@@ -64,7 +67,7 @@ public class TikaProcessorTest {
}
@Test
- public void UTF16_input() throws Exception {
+ void UTF16_input() throws Exception {
Document doc = mkDocument(Resources.getResourceStream("/binaries/UTF16_with_signature.xml"),
DocumentNameMatcher.MATCHES_ALL);
TikaProcessor.process(doc);
@@ -80,48 +83,48 @@ public class TikaProcessorTest {
}
@Test
- public void UTF8_input() throws Exception {
+ void UTF8_input() throws Exception {
FileDocument doc = mkDocument("/binaries/UTF8_with_signature.xml");
TikaProcessor.process(doc);
assertEquals(Document.Type.STANDARD, doc.getMetaData().getDocumentType());
}
@Test
- public void RAT178Test() {
+ void RAT178Test() {
FileDocument doc = new FileDocument(new File("/not_a_real_file"), DocumentNameMatcher.MATCHES_ALL);
assertThrows(RatDocumentAnalysisException.class, () ->TikaProcessor.process(doc));
}
@Test
- public void missNamedBinaryTest() throws Exception {
+ void missNamedBinaryTest() throws Exception {
FileDocument doc = mkDocument("/binaries/Image-png.not");
TikaProcessor.process(doc);
assertEquals(Document.Type.BINARY, doc.getMetaData().getDocumentType());
}
@Test
- public void plainTextTest() throws Exception {
+ void plainTextTest() throws Exception {
FileDocument doc = mkDocument(Resources.getExampleResource("exampleData/Text.txt"));
TikaProcessor.process(doc);
assertEquals(Document.Type.STANDARD, doc.getMetaData().getDocumentType());
}
@Test
- public void emptyFileTest() throws Exception {
+ void emptyFileTest() throws Exception {
FileDocument doc = mkDocument(Resources.getExampleResource("exampleData/sub/Empty.txt"));
TikaProcessor.process(doc);
assertEquals(Document.Type.STANDARD, doc.getMetaData().getDocumentType());
}
@Test
- public void javaFileWithChineseCharacters_RAT301() throws Exception {
+ void javaFileWithChineseCharacters_RAT301() throws Exception {
FileDocument doc = mkDocument("/tikaFiles/standard/ChineseCommentsJava.java");
TikaProcessor.process(doc);
assertEquals(Document.Type.STANDARD, doc.getMetaData().getDocumentType());
}
@Test
- public void testTikaFiles() throws RatDocumentAnalysisException {
+ void testTikaFiles() throws RatDocumentAnalysisException {
File dir = new File("src/test/resources/tikaFiles");
Map<String, Document.Type> unseenMime = TikaProcessor.getDocumentTypeMap();
ClaimStatistic statistic = new ClaimStatistic();
@@ -144,6 +147,22 @@ public class TikaProcessorTest {
}
}
+ @Test
+ void testDetectionOfInvalidData() throws IOException {
+ byte[] invalidData = new byte[] {
+ 0x00, (byte) 0xFF, 0x00, (byte) 0xFE,
+ 0x01, (byte) 0x80, 0x00, 0x7F
+ };
+ // as Tika works with a probabilistic encoding detection it does not return NO encoding
+ assertThat(TikaProcessor.detectCharset(new ByteArrayInputStream(invalidData), null)).isEqualTo(Charset.forName("Windows-1258"));
+ }
+
+ @Test
+ void testEmptyFileEncoding() throws IOException {
+ byte[] empty = {};
+ assertThat(TikaProcessor.detectCharset(new ByteArrayInputStream(empty), null)).isNull();
+ }
+
/**
* Build a document with the specific input stream
* @return a document with the specific input stream
diff --git a/apache-rat-core/src/test/java/org/apache/rat/document/DocumentNameTest.java b/apache-rat-core/src/test/java/org/apache/rat/document/DocumentNameTest.java
index 5abac9625..7b1ef2489 100644
--- a/apache-rat-core/src/test/java/org/apache/rat/document/DocumentNameTest.java
+++ b/apache-rat-core/src/test/java/org/apache/rat/document/DocumentNameTest.java
@@ -172,7 +172,6 @@ public class DocumentNameTest {
String resolveName2 = fsInfo.roots()[0] + fsInfo.mkPath("dir", fsInfo.toString(), "thing");
assertThat(base.resolve(resolveName2).getName())
.isEqualTo(resolveName2);
-
}
void testNoRootSpecified() {
@@ -258,7 +257,6 @@ public class DocumentNameTest {
}
}
-
@ParameterizedTest(name = "{index} {0} {1}")
@MethodSource("archiveEntryTestData")
void archiveEntryNameTest(String os, String testName, DocumentName archiveName, String root, String separator, String baseName,
@@ -300,4 +298,5 @@ public class DocumentNameTest {
}
return lst;
}
+
}
diff --git a/apache-rat-core/src/test/java/org/apache/rat/document/FileDocumentTest.java b/apache-rat-core/src/test/java/org/apache/rat/document/FileDocumentTest.java
index a36944f4e..93fac968d 100644
--- a/apache-rat-core/src/test/java/org/apache/rat/document/FileDocumentTest.java
+++ b/apache-rat-core/src/test/java/org/apache/rat/document/FileDocumentTest.java
@@ -18,9 +18,11 @@
*/
package org.apache.rat.document;
+import java.io.IOException;
import java.nio.file.Path;
import org.apache.commons.io.FileUtils;
import org.apache.rat.api.Document;
+import org.apache.rat.api.MetaData;
import org.apache.rat.test.utils.Resources;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -31,6 +33,10 @@ import java.io.Reader;
import org.junit.jupiter.api.io.TempDir;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
public class FileDocumentTest {
private Document document;
@@ -64,4 +70,23 @@ public class FileDocumentTest {
final DocumentName name = document.getName();
assertThat(name).isNotNull();
}
+
+ /**
+ * As MetaData is set within the constructor of the underlying document,
+ * we use a combination of spy and mock to explicitly cover this code branch.
+ */
+ @Test
+ void handleFlowIfNoEncodingFound() {
+ Document spyDocument = spy(document);
+ MetaData mockMetaData = mock(MetaData.class);
+
+ when(spyDocument.getMetaData()).thenReturn(mockMetaData);
+ assertThat(spyDocument.getMetaData()).isSameAs(mockMetaData);
+
+ IOException expectedException = assertThrows(IOException.class, spyDocument::reader);
+ assertThat(expectedException.getMessage()).startsWith("No charset detected for document");
+
+ // just raise test coverage and test uncovered branch
+ assertThat(spyDocument.equals(document.getName())).isFalse();
+ }
}
diff --git a/apache-rat-plugin/src/it/it1/verify.groovy b/apache-rat-plugin/src/it/it1/verify.groovy
index 3f0d20bc8..275a4aa74 100644
--- a/apache-rat-plugin/src/it/it1/verify.groovy
+++ b/apache-rat-plugin/src/it/it1/verify.groovy
@@ -35,7 +35,7 @@ Document document = XmlUtils.toDom(new FileInputStream(f))
XPath xPath = XPathFactory.newInstance().newXPath()
XmlUtils.assertAttributes(document, xPath, "/rat-report/resource[@name='/src.apt']",
- mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain", "type", "STANDARD" ))
+ mapOf("encoding", "windows-1252", "mediaType", "text/plain", "type", "STANDARD" ))
XmlUtils.assertAttributes(document, xPath, "/rat-report/resource[@name='/src.apt']/license[@id='MyLicense']",
mapOf("approval", "true", "family", "YAL ", "name", "Yet another license" ))
diff --git a/apache-rat-plugin/src/test/java/org/apache/rat/mp/RatCheckMojoTest.java b/apache-rat-plugin/src/test/java/org/apache/rat/mp/RatCheckMojoTest.java
index 5734ab73e..4c5a3f0bf 100644
--- a/apache-rat-plugin/src/test/java/org/apache/rat/mp/RatCheckMojoTest.java
+++ b/apache-rat-plugin/src/test/java/org/apache/rat/mp/RatCheckMojoTest.java
@@ -153,7 +153,7 @@ public class RatCheckMojoTest {
mapOf("mediaType", "application/octet-stream", "type", "IGNORED"));
XmlUtils.assertAttributes(document, xPath, "/rat-report/resource[@name='/pom.xml']",
- mapOf("mediaType", "application/xml", "type", "STANDARD", "encoding", "ISO-8859-1"));
+ mapOf("mediaType", "application/xml", "type", "STANDARD", "encoding", "windows-1252"));
}
private static Map<String, String> mapOf(String... parts) {
@@ -291,7 +291,7 @@ public class RatCheckMojoTest {
XmlUtils.assertAttributes(document, xPath, "/rat-report/resource[@name='/pom.xml']",
"mediaType", "application/xml", "type", "IGNORED", "isDirectory", "false");
XmlUtils.assertAttributes(document, xPath, "/rat-report/resource[@name='/src/main/java/nl/basjes/something/Something.java']",
- "mediaType", "text/x-java-source", "type", "STANDARD", "encoding", "ISO-8859-1");
+ "mediaType", "text/x-java-source", "type", "STANDARD", "encoding", "windows-1252");
XmlUtils.assertAttributes(document, xPath, "/rat-report/resource[@name='/src/main/java/nl/basjes/something/Something.java']/license",
"approval", "true", "family", ILicenseFamily.makeCategory("CC"), "id", "CC-BY-NC-ND", "name",
"Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International");
diff --git a/apache-rat-tasks/src/test/resources/antunit/report-normal-operation.xml b/apache-rat-tasks/src/test/resources/antunit/report-normal-operation.xml
index 227ad7cec..d3b2a2ad5 100644
--- a/apache-rat-tasks/src/test/resources/antunit/report-normal-operation.xml
+++ b/apache-rat-tasks/src/test/resources/antunit/report-normal-operation.xml
@@ -74,7 +74,7 @@ SPDX-License-Identifier: Apache-2.0
</path>
</pathconvert>
<property name="expectedOutputXML"
- value='<resource encoding="ISO-8859-1" mediaType="application/xml" name="/report-normal-operation.xml" type="STANDARD"' />
+ value='<resource encoding="windows-1252" mediaType="application/xml" name="/report-normal-operation.xml" type="STANDARD"' />
<property name="expectedOutputXML2" value='family="AL "' />
</target>
diff --git a/pom.xml b/pom.xml
index 52ee993f8..b65e3e5c3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -51,7 +51,7 @@ agnostic home for software distribution comprehension and audit tools.
<assertj.version>4.0.0-M1</assertj.version>
<!-- this is the target Java version that this project supports at runtime -->
<javaVersion>17</javaVersion>
- <tika.version>3.3.2</tika.version>
+ <tika.version>4.0.0</tika.version>
<mockito.version>5.23.0</mockito.version>
<!-- This is the version of Maven required to use the RAT Maven Plugin -->
<mavenMinVersion>3.9</mavenMinVersion>
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 9df1dd1cc..749b81b67 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -68,6 +68,9 @@ in order to be properly linked in site reports.
</release>
-->
<release version="1.0.0-SNAPSHOT" date="xxxx-yy-zz" description="Current SNAPSHOT - release to be done">
+ <action issue="RAT-532" type="add" dev="pottlinger">
+ Update to Tika 4.0.0: new charset detection logic in Tika returns different values compared to 3.x before, such as windows-1252 instead of ISO-8859-1.
+ </action>
<action issue="RAT-541" type="add" dev="claudenw">
Refactor UIOption, AntOption and MavenOption to improve multi-threaded usage and provide a clearer separation of responsibilities.
</action>