You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/09/30 15:06:31 UTC
[tika] branch main updated (f90b64a -> dd9acd2)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git.
from f90b64a Merge remote-tracking branch 'origin/main' into main
new 2f010f8 add prefix to S3Fetcher
new dd9acd2 TIKA-3562 -- add jxl detection
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.txt | 2 ++
.../org/apache/tika/mime/tika-mimetypes.xml | 9 +++++++
.../src/test/resources/test-documents/testJXL.jxl | Bin 0 -> 81 bytes
.../java/org/apache/tika/mime/TestMimeTypes.java | 8 ++++++
.../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 27 +++++++++++++++++----
5 files changed, 41 insertions(+), 5 deletions(-)
create mode 100644 tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/resources/test-documents/testJXL.jxl
[tika] 02/02: TIKA-3562 -- add jxl detection
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit dd9acd2a65fc4e31881784bfae8150bbaa32be14
Author: tballison <ta...@apache.org>
AuthorDate: Thu Sep 30 11:06:22 2021 -0400
TIKA-3562 -- add jxl detection
---
CHANGES.txt | 2 ++
.../main/resources/org/apache/tika/mime/tika-mimetypes.xml | 9 +++++++++
.../src/test/resources/test-documents/testJXL.jxl | Bin 0 -> 81 bytes
.../src/test/java/org/apache/tika/mime/TestMimeTypes.java | 8 ++++++++
4 files changed, 19 insertions(+)
diff --git a/CHANGES.txt b/CHANGES.txt
index 37990f1..c670675 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,7 @@
Release 2.1.1 - ???
+ * Add detection of JPEG XL file types (TIKA-3562)
+
* Improve robustness and features of the httpfetcher (TIKA-3543)
* Add optional fetch ranges to FetchEmitTuple to allow range fetching from,
diff --git a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
index 89123f7..8c337a3 100644
--- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
+++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
@@ -6058,6 +6058,15 @@
</magic>
</mime-type>
+ <mime-type type="image/jxl">
+ <_comment>JPEG XL</_comment>
+ <magic priority="50">
+ <match value="0xFF0A" type="string" offset="0"/>
+ <match value="0x0000000C4A584C200D0A870A" type="string" offset="0"/>
+ </magic>
+ <glob pattern="*.jxl"/>
+ </mime-type>
+
<mime-type type="image/x-niff">
<_comment>Navy Interchange File Format</_comment>
<magic priority="50">
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/resources/test-documents/testJXL.jxl b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/resources/test-documents/testJXL.jxl
new file mode 100644
index 0000000..ced8bfa
Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/resources/test-documents/testJXL.jxl differ
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/mime/TestMimeTypes.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/mime/TestMimeTypes.java
index 18f71a4..308f31a 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/mime/TestMimeTypes.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/mime/TestMimeTypes.java
@@ -432,6 +432,14 @@ public class TestMimeTypes {
}
@Test
+ public void testJxlDetection() throws Exception {
+ //test file created by: https://github.com/surma/jxl-art/blob/main/LICENSE
+ assertType("image/jxl", "testJXL.jxl");
+ assertTypeByData("image/jxl", "testJXL.jxl");
+ assertTypeByName("image/jxl", "testJXL.jxl");
+ }
+
+ @Test
public void testAVIFDetection() throws Exception {
// The test file is an avif header fragment only, not a complete image.
assertType("image/avif", "testAVIF.avif");
[tika] 01/02: add prefix to S3Fetcher
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 2f010f877272fd881634b82fb8a00d681211eca5
Author: tballison <ta...@apache.org>
AuthorDate: Thu Sep 30 10:59:12 2021 -0400
add prefix to S3Fetcher
---
.../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 27 ++++++++++++++++++----
1 file changed, 22 insertions(+), 5 deletions(-)
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
index 5f400dd..0e8b190 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java
@@ -45,6 +45,7 @@ import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.pipes.fetcher.AbstractFetcher;
import org.apache.tika.pipes.fetcher.RangeFetcher;
+import org.apache.tika.utils.StringUtils;
/**
* Fetches files from s3. Example string: s3://my_bucket/path/to/my_file.pdf
@@ -58,6 +59,7 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe
private String region;
private String bucket;
private String profile;
+ private String prefix;
private String credentialsProvider;
private boolean extractUserMetadata = true;
private int maxConnections = ClientConfiguration.DEFAULT_MAX_CONNECTIONS;
@@ -67,10 +69,11 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe
@Override
public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
- LOGGER.debug("about to fetch fetchkey={} from bucket ({})", fetchKey, bucket);
+ String theFetchKey = StringUtils.isBlank(prefix) ? fetchKey : prefix + fetchKey;
+ LOGGER.debug("about to fetch fetchkey={} from bucket ({})", theFetchKey, bucket);
try {
- S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, fetchKey));
+ S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, theFetchKey));
if (extractUserMetadata) {
for (Map.Entry<String, String> e : s3Object.getObjectMetadata().getUserMetadata()
.entrySet()) {
@@ -95,13 +98,14 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe
@Override
public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata)
throws TikaException, IOException {
+ String theFetchKey = StringUtils.isBlank(prefix) ? fetchKey : prefix + fetchKey;
//TODO -- figure out how to integrate this
- LOGGER.debug("about to fetch fetchkey={} (start={} end={}) from bucket ({})", fetchKey,
- startRange, endRange, bucket);
+ LOGGER.debug("about to fetch fetchkey={} (start={} end={}) from bucket ({})",
+ theFetchKey, startRange, endRange, bucket);
try {
S3Object s3Object = s3Client.getObject(
- new GetObjectRequest(bucket, fetchKey).withRange(startRange, endRange));
+ new GetObjectRequest(bucket, theFetchKey).withRange(startRange, endRange));
if (extractUserMetadata) {
for (Map.Entry<String, String> e : s3Object.getObjectMetadata().getUserMetadata()
@@ -145,6 +149,19 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe
}
/**
+ * prefix to prepend to the fetch key before fetching.
+ * This will automatically add a '/' at the end.
+ * @param prefix
+ */
+ @Field
+ public void setPrefix(String prefix) {
+ //guarantee that the prefix ends with /
+ if (! prefix.endsWith("/")) {
+ prefix += "/";
+ }
+ this.prefix = prefix;
+ }
+ /**
* Whether or not to extract user metadata from the S3Object
*
* @param extractUserMetadata