You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ti...@apache.org on 2025/06/08 11:45:35 UTC
(tika) branch branch_3x updated (1d7834d10 -> d73e6b01a)
This is an automated email from the ASF dual-hosted git repository.
tilman pushed a change to branch branch_3x
in repository https://gitbox.apache.org/repos/asf/tika.git
from 1d7834d10 TIKA-4435: set a region
new fdd0eb513 TIKA-4435: add more aws components, add jackson json, jaxb (for aws)
new ece119964 TIKA-4435: delete files from bucket; replace deprecated SelectObject with json; add debugging
new d73e6b01a TIKA-4435: adjust some expected test results
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../tika-parsers-ml/tika-transcribe-aws/pom.xml | 31 ++++-
.../parser/transcribe/aws/AmazonTranscribe.java | 125 ++++++++-------------
.../transcribe/aws/AmazonTranscribeTest.java | 22 ++--
3 files changed, 89 insertions(+), 89 deletions(-)
(tika) 03/03: TIKA-4435: adjust some expected test results
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tilman pushed a commit to branch branch_3x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit d73e6b01a28dfd5f767ed7805e7c4450a9b8dbbf
Author: Tilman Hausherr <ti...@apache.org>
AuthorDate: Sun Jun 8 13:43:30 2025 +0200
TIKA-4435: adjust some expected test results
---
.../transcribe/aws/AmazonTranscribeTest.java | 22 +++++++++++++++-------
1 file changed, 15 insertions(+), 7 deletions(-)
diff --git a/tika-parsers/tika-parsers-ml/tika-transcribe-aws/src/test/java/org/apache/tika/parser/transcribe/aws/AmazonTranscribeTest.java b/tika-parsers/tika-parsers-ml/tika-transcribe-aws/src/test/java/org/apache/tika/parser/transcribe/aws/AmazonTranscribeTest.java
index 6bdee4015..4ff112c2b 100644
--- a/tika-parsers/tika-parsers-ml/tika-transcribe-aws/src/test/java/org/apache/tika/parser/transcribe/aws/AmazonTranscribeTest.java
+++ b/tika-parsers/tika-parsers-ml/tika-transcribe-aws/src/test/java/org/apache/tika/parser/transcribe/aws/AmazonTranscribeTest.java
@@ -35,8 +35,16 @@ import org.apache.tika.parser.Parser;
* Tests tika-trancribe by creating an AmazonTranscribe() object.
* 1) Tests that transcribe functions properly when it is given just a filepath.
* 2) Both audio (mp3) and video (mp4) files are used in these tests.
+ *
+ * How to get this to work:
+ * 1) remove "@Disabled"
+ * 2) Get an amazon aws account (preferably the free tier)
+ * 3) include access key (clientid), secret and bucket name
+ * in "tika-config-aws-transcribe.xml" (do not commit this file!). The bucket name must be unique worldwide.
+ * 4) Make sure you have the needed permissions policies, AmazonS3FullAccess and AmazonTranscribeFullAccess.
+ * 5) Be aware that as of 6/2025, you can get only 60 min / month free transscripts.
*/
-@Disabled("Ignore until finalize AmazonTrancsribe Interface & build Tika")
+@Disabled("Ignore until finalize AmazonTranscribe Interface & build Tika")
public class AmazonTranscribeTest extends TikaTest {
static Parser PARSER;
@@ -58,7 +66,7 @@ public class AmazonTranscribeTest extends TikaTest {
ParseContext context = new ParseContext();
context.set(LanguageCode.class, LanguageCode.EnUS);
String xml = getXML("en-US_(A_Little_Bottle_Of_Water).mp3", PARSER, context).xml;
- String expected = "a little bottle of water.";
+ String expected = "A little bottle of water";
assertContains(expected, xml);
}
@@ -69,7 +77,7 @@ public class AmazonTranscribeTest extends TikaTest {
@Test
public void testAmazonTranscribeUnknownAudio_enUS() throws Exception {
String xml = getXML("en-US_(A_Little_Bottle_Of_Water).mp3", PARSER).xml;
- String expected = "a little bottle of water.";
+ String expected = "A little bottle of water";
assertContains(expected, xml);
}
@@ -104,7 +112,7 @@ public class AmazonTranscribeTest extends TikaTest {
@Test
public void testAmazonTranscribeAudio_enGB() throws Exception {
String file = "en-GB_(A_Little_Bottle_Of_Water).mp3";
- String expected = "a little bottle of water.";
+ String expected = "A little bottle of water";
ParseContext context = new ParseContext();
context.set(LanguageCode.class, LanguageCode.EnGB);
String xml = getXML(file, PARSER, context).xml;
@@ -118,7 +126,7 @@ public class AmazonTranscribeTest extends TikaTest {
@Test
public void testAmazonTranscribeUnknownAudio_enGB() throws Exception {
String file = "en-GB_(A_Little_Bottle_Of_Water).mp3";
- String expected = "a little bottle of water.";
+ String expected = "A little bottle of water";
String xml = getXML(file, PARSER).xml;
assertContains(expected, xml);
}
@@ -130,7 +138,7 @@ public class AmazonTranscribeTest extends TikaTest {
@Test
public void testAmazonTranscribeAudio_enAU() throws Exception {
String file = "en-AU_(A_Little_Bottle_Of_Water).mp3";
- String expected = "a little bottle of water.";
+ String expected = "A little bottle of water";
ParseContext context = new ParseContext();
context.set(LanguageCode.class, LanguageCode.EnAU);
String xml = getXML(file, PARSER, context).xml;
@@ -144,7 +152,7 @@ public class AmazonTranscribeTest extends TikaTest {
@Test
public void testAmazonTranscribeUnknownAudio_enAU() throws Exception {
String file = "en-AU_(A_Little_Bottle_Of_Water).mp3";
- String expected = "a little bottle of water.";
+ String expected = "A little bottle of water";
String xml = getXML(file, PARSER).xml;
assertContains(expected, xml);
}
(tika) 02/03: TIKA-4435: delete files from bucket; replace deprecated SelectObject with json; add debugging
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tilman pushed a commit to branch branch_3x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit ece1199649de02de0b241dc2ae1d7dae3659ea68
Author: Tilman Hausherr <ti...@apache.org>
AuthorDate: Sun Jun 8 13:36:42 2025 +0200
TIKA-4435: delete files from bucket; replace deprecated SelectObject with json; add debugging
---
.../parser/transcribe/aws/AmazonTranscribe.java | 125 ++++++++-------------
1 file changed, 44 insertions(+), 81 deletions(-)
diff --git a/tika-parsers/tika-parsers-ml/tika-transcribe-aws/src/main/java/org/apache/tika/parser/transcribe/aws/AmazonTranscribe.java b/tika-parsers/tika-parsers-ml/tika-transcribe-aws/src/main/java/org/apache/tika/parser/transcribe/aws/AmazonTranscribe.java
index fb20f2522..db5600232 100644
--- a/tika-parsers/tika-parsers-ml/tika-transcribe-aws/src/main/java/org/apache/tika/parser/transcribe/aws/AmazonTranscribe.java
+++ b/tika-parsers/tika-parsers-ml/tika-transcribe-aws/src/main/java/org/apache/tika/parser/transcribe/aws/AmazonTranscribe.java
@@ -17,19 +17,14 @@
package org.apache.tika.parser.transcribe.aws;
-import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
@@ -38,19 +33,15 @@ import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.CompressionType;
-import com.amazonaws.services.s3.model.ExpressionType;
-import com.amazonaws.services.s3.model.InputSerialization;
-import com.amazonaws.services.s3.model.JSONInput;
-import com.amazonaws.services.s3.model.JSONOutput;
-import com.amazonaws.services.s3.model.JSONType;
-import com.amazonaws.services.s3.model.OutputSerialization;
+import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.SelectObjectContentEvent;
-import com.amazonaws.services.s3.model.SelectObjectContentEventVisitor;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.SelectObjectContentResult;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
+import com.amazonaws.services.securitytoken.model.GetCallerIdentityRequest;
+import com.amazonaws.services.securitytoken.model.GetCallerIdentityResult;
import com.amazonaws.services.transcribe.AmazonTranscribeAsync;
import com.amazonaws.services.transcribe.AmazonTranscribeAsyncClientBuilder;
import com.amazonaws.services.transcribe.model.GetTranscriptionJobRequest;
@@ -60,9 +51,8 @@ import com.amazonaws.services.transcribe.model.Media;
import com.amazonaws.services.transcribe.model.StartTranscriptionJobRequest;
import com.amazonaws.services.transcribe.model.TranscriptionJob;
import com.amazonaws.services.transcribe.model.TranscriptionJobStatus;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.ContentHandler;
@@ -100,7 +90,7 @@ public class AmazonTranscribe implements Parser, Initializable {
private String bucketName;
private String region;
private boolean isAvailable; // Flag for whether or not transcription is available.
- private String clientId;
+ private String clientId; // Access key
private String clientSecret; // Keys used for the API calls.
private AWSStaticCredentialsProvider credsProvider;
@@ -168,6 +158,7 @@ public class AmazonTranscribe implements Parser, Initializable {
xhtml.endElement("p");
xhtml.endDocument();
+ deleteFilesFromBucket(jobName);
}
@@ -252,7 +243,16 @@ public class AmazonTranscribe implements Parser, Initializable {
try {
@SuppressWarnings("unused") PutObjectResult response = amazonS3.putObject(request);
} catch (SdkClientException e) {
- throw (new TikaException("File Upload to AWS Failed"));
+ throw new TikaException("File upload to AWS failed: " + e.getMessage(), e);
+ }
+ }
+
+ private void deleteFilesFromBucket(String jobName) throws TikaException {
+ try {
+ amazonS3.deleteObject(bucketName, jobName);
+ amazonS3.deleteObject(bucketName, jobName + ".json");
+ } catch (SdkClientException e) {
+ LOG.error("Failed to delete {} and/or {} from {}", jobName, jobName + ".json", bucketName, e);
}
}
@@ -272,69 +272,24 @@ public class AmazonTranscribe implements Parser, Initializable {
private String getTranscriptText(String fileNameS3)
throws AmazonServiceException, SdkClientException, IOException {
TranscriptionJob transcriptionJob = retrieveObjectWhenJobCompleted(fileNameS3);
- String text = null;
+ String text = "";
if (transcriptionJob != null && !TranscriptionJobStatus.FAILED.name()
.equals(transcriptionJob.getTranscriptionJobStatus())) {
- InputSerialization inputSerialization =
- new InputSerialization().withJson(new JSONInput().withType(JSONType.DOCUMENT))
- .withCompressionType(CompressionType.NONE);
- OutputSerialization outputSerialization =
- new OutputSerialization().withJson(new JSONOutput());
- SelectObjectContentRequest request =
- new SelectObjectContentRequest().withBucketName(this.bucketName)
- .withKey(fileNameS3 + ".json").withExpression(
- "Select s.results.transcripts[0].transcript from S3Object s")
- //WHERE transcript IS NOT MISSING
- .withExpressionType(ExpressionType.SQL)
- .withRequestCredentialsProvider(credsProvider);
- request.setInputSerialization(inputSerialization);
- request.setOutputSerialization(outputSerialization);
-
- final AtomicBoolean isResultComplete = new AtomicBoolean(false);
-
- try (SelectObjectContentResult result = amazonS3.selectObjectContent(request)) {
- InputStream resultInputStream = result.getPayload()
- .getRecordsInputStream(new SelectObjectContentEventVisitor() {
- @Override
- public void visit(SelectObjectContentEvent.StatsEvent event) {
- LOG.debug("Received Stats, Bytes Scanned: " +
- event.getDetails().getBytesScanned() +
- " Bytes Processed: " +
- event.getDetails().getBytesProcessed());
- }
-
- /*
- * An End Event informs that the request has
- * finished successfully.
- */
- @Override
- public void visit(SelectObjectContentEvent.EndEvent event) {
- isResultComplete.set(true);
- LOG.debug("Received End Event. Result is complete.");
- }
- });
- text = new BufferedReader(
- new InputStreamReader(resultInputStream, StandardCharsets.UTF_8)).lines()
- .collect(Collectors.joining("\n"));
- }
- /*
- * The End Event indicates all matching records have been
- * transmitted. If the End Event is not received, the results
- * may be incomplete.
- */
- if (!isResultComplete.get()) {
- throw new IOException(
- "S3 Select request was incomplete as End Event was not received.");
+ S3Object s3Object = amazonS3.getObject(new GetObjectRequest(bucketName, fileNameS3 + ".json"));
+ try (S3ObjectInputStream objectContent = s3Object.getObjectContent()) {
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode root = mapper.readTree(objectContent);
+ text = root
+ .path("results")
+ .path("transcripts")
+ .get(0)
+ .path("transcript")
+ .asText();
+ // could also be done with json.simple:
+ // ((JSONObject)((JSONArray)((JSONObject) obj.get("results")).get("transcripts")).get(0)).get("transcript")
}
}
- JSONParser parser = new JSONParser();
- JSONObject obj = null;
- try {
- obj = (JSONObject) parser.parse(text);
- } catch (ParseException e) {
- throw new IOException(e.getMessage(), e);
- }
- return obj.get("transcript").toString();
+ return text;
}
/**
@@ -375,7 +330,15 @@ public class AmazonTranscribe implements Parser, Initializable {
AmazonS3ClientBuilder.standard().withCredentials(credsProvider).build();
this.region = amazonS3.getRegionName(); // not sure if this works at all
}
- if (!this.amazonS3.doesBucketExistV2(this.bucketName)) {
+
+ // for debugging
+ AWSSecurityTokenService stsClient = AWSSecurityTokenServiceClientBuilder.standard()
+ .withCredentials(credsProvider).withRegion(region)
+ .build();
+ GetCallerIdentityResult identity = stsClient.getCallerIdentity(new GetCallerIdentityRequest());
+ LOG.debug("Authenticated as: {}", identity.getArn());
+
+ if (!this.amazonS3.doesBucketExistV2(this.bucketName)) { // returns true if no access
try {
amazonS3.createBucket(this.bucketName);
} catch (AmazonS3Exception e) {
(tika) 01/03: TIKA-4435: add more aws components, add jackson json, jaxb (for aws)
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tilman pushed a commit to branch branch_3x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit fdd0eb5134ca923fc47c2d835066e5047769e362
Author: Tilman Hausherr <ti...@apache.org>
AuthorDate: Sun Jun 8 13:34:22 2025 +0200
TIKA-4435: add more aws components, add jackson json, jaxb (for aws)
---
.../tika-parsers-ml/tika-transcribe-aws/pom.xml | 31 +++++++++++++++++++++-
1 file changed, 30 insertions(+), 1 deletion(-)
diff --git a/tika-parsers/tika-parsers-ml/tika-transcribe-aws/pom.xml b/tika-parsers/tika-parsers-ml/tika-transcribe-aws/pom.xml
index 90fae4580..cbb3a6c0a 100644
--- a/tika-parsers/tika-parsers-ml/tika-transcribe-aws/pom.xml
+++ b/tika-parsers/tika-parsers-ml/tika-transcribe-aws/pom.xml
@@ -32,6 +32,17 @@
<packaging>bundle</packaging>
<name>Apache Tika transcribe aws</name>
<url>http://tika.apache.org/</url>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-swf-libraries</artifactId>
+ <version>2.0.0</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<!--TODO use latest aws version or the one defined in the tika-parent-->
<dependencies>
<dependency>
@@ -42,7 +53,6 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
- <version>${aws.version}</version>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
@@ -53,6 +63,25 @@
<artifactId>slf4j-simple</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>${aws.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-simpleworkflow</artifactId>
+ <version>${aws.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ <version>2.3.1</version>
+ </dependency>
</dependencies>
<build>
<plugins>