You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2022/11/16 16:44:48 UTC
[tika] branch main updated (852aa3051 -> b7499a3ef)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
from 852aa3051 Merge pull request #798 from apache/dependabot/maven/aws.version-1.12.343
new 7988f9baa TIKA-3917 -- general upgrades for 2.6.1
new b7499a3ef improve robustness of AsyncProcessor
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/tika/pipes/async/AsyncProcessor.java | 64 +++++++++++-----------
.../opensearch/tests/TikaPipesOpenSearchTest.java | 2 +-
.../apache/tika/async/cli/TikaAsyncCLITest.java | 25 +++++----
.../src/test/resources/tika-config-broken.xml | 24 ++++----
4 files changed, 62 insertions(+), 53 deletions(-)
copy tika-batch/src/test/java/org/apache/tika/batch/fs/FSUtilTest.java => tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaAsyncCLITest.java (59%)
copy tika-core/src/test/resources/org/apache/tika/pipes/async/TIKA-3507.xml => tika-pipes/tika-async-cli/src/test/resources/tika-config-broken.xml (66%)
[tika] 01/02: TIKA-3917 -- general upgrades for 2.6.1
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 7988f9baa3968dff6a399547aba98536573f1e80
Author: tballison <ta...@apache.org>
AuthorDate: Wed Nov 16 08:52:04 2022 -0500
TIKA-3917 -- general upgrades for 2.6.1
---
.../org/apache/tika/pipes/opensearch/tests/TikaPipesOpenSearchTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tika-integration-tests/tika-pipes-opensearch-integration-tests/src/test/java/org/apache/tika/pipes/opensearch/tests/TikaPipesOpenSearchTest.java b/tika-integration-tests/tika-pipes-opensearch-integration-tests/src/test/java/org/apache/tika/pipes/opensearch/tests/TikaPipesOpenSearchTest.java
index f7771be07..838062045 100644
--- a/tika-integration-tests/tika-pipes-opensearch-integration-tests/src/test/java/org/apache/tika/pipes/opensearch/tests/TikaPipesOpenSearchTest.java
+++ b/tika-integration-tests/tika-pipes-opensearch-integration-tests/src/test/java/org/apache/tika/pipes/opensearch/tests/TikaPipesOpenSearchTest.java
@@ -31,7 +31,7 @@ import org.apache.tika.pipes.xsearch.tests.TikaPipesXSearchBase;
@Testcontainers(disabledWithoutDocker = true)
public class TikaPipesOpenSearchTest extends TikaPipesXSearchBase {
- private static final String DOCKER_IMAGE_NAME = "opensearchproject/opensearch:2.3.0";
+ private static final String DOCKER_IMAGE_NAME = "opensearchproject/opensearch:2.4.0";
@Container
public static GenericContainer<?> OPEN_SEARCH_CONTAINER =
[tika] 02/02: improve robustness of AsyncProcessor
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit b7499a3ef2276ebc0c910d27b198f46d0b2d2f8d
Author: tballison <ta...@apache.org>
AuthorDate: Wed Nov 16 11:44:34 2022 -0500
improve robustness of AsyncProcessor
---
.../apache/tika/pipes/async/AsyncProcessor.java | 64 +++++++++++-----------
.../apache/tika/async/cli/TikaAsyncCLITest.java | 44 +++++++++++++++
.../src/test/resources/tika-config-broken.xml | 36 ++++++++++++
3 files changed, 113 insertions(+), 31 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java b/tika-core/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java
index e86aa71b3..476e4df58 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/async/AsyncProcessor.java
@@ -82,41 +82,43 @@ public class AsyncProcessor implements Closeable {
asyncConfig.getNumClients() + asyncConfig.getNumEmitters() + 1);
this.executorCompletionService =
new ExecutorCompletionService<>(executorService);
- if (!tikaConfigPath.toAbsolutePath()
- .equals(asyncConfig.getTikaConfig().toAbsolutePath())) {
- LOG.warn("TikaConfig for AsyncProcessor ({}) is different " +
- "from TikaConfig for workers ({}). If this is intended," +
- " please ignore this warning.",
- tikaConfigPath.toAbsolutePath(),
- asyncConfig.getTikaConfig().toAbsolutePath()
- );
- }
- this.executorCompletionService.submit(() -> {
- while (true) {
- try {
- Thread.sleep(500);
- checkActive();
- } catch (InterruptedException e) {
- return WATCHER_FUTURE_CODE;
+ try {
+ if (!tikaConfigPath.toAbsolutePath().equals(asyncConfig.getTikaConfig().toAbsolutePath())) {
+ LOG.warn("TikaConfig for AsyncProcessor ({}) is different " +
+ "from TikaConfig for workers ({}). If this is intended," +
+ " please ignore this warning.", tikaConfigPath.toAbsolutePath(),
+ asyncConfig.getTikaConfig().toAbsolutePath());
+ }
+ this.executorCompletionService.submit(() -> {
+ while (true) {
+ try {
+ Thread.sleep(500);
+ checkActive();
+ } catch (InterruptedException e) {
+ return WATCHER_FUTURE_CODE;
+ }
}
+ });
+ //this is run in a daemon thread
+ if (pipesIterator != null && (pipesIterator instanceof TotalCounter)) {
+ LOG.debug("going to total counts");
+ startCounter((TotalCounter) pipesIterator);
}
- });
- //this is run in a daemon thread
- if (pipesIterator != null &&
- (pipesIterator instanceof TotalCounter)) {
- LOG.debug("going to total counts");
- startCounter((TotalCounter) pipesIterator);
- }
- for (int i = 0; i < asyncConfig.getNumClients(); i++) {
- executorCompletionService.submit(new FetchEmitWorker(asyncConfig, fetchEmitTuples,
- emitData));
- }
+ for (int i = 0; i < asyncConfig.getNumClients(); i++) {
+ executorCompletionService.submit(
+ new FetchEmitWorker(asyncConfig, fetchEmitTuples, emitData));
+ }
- EmitterManager emitterManager = EmitterManager.load(asyncConfig.getTikaConfig());
- for (int i = 0; i < asyncConfig.getNumEmitters(); i++) {
- executorCompletionService.submit(new AsyncEmitter(asyncConfig, emitData,
- emitterManager));
+ EmitterManager emitterManager = EmitterManager.load(asyncConfig.getTikaConfig());
+ for (int i = 0; i < asyncConfig.getNumEmitters(); i++) {
+ executorCompletionService.submit(
+ new AsyncEmitter(asyncConfig, emitData, emitterManager));
+ }
+ } catch (Exception e) {
+ executorService.shutdownNow();
+ asyncConfig.getPipesReporter().close();
+ throw e;
}
}
diff --git a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaAsyncCLITest.java b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaAsyncCLITest.java
new file mode 100644
index 000000000..a06f71c39
--- /dev/null
+++ b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaAsyncCLITest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.async.cli;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.utils.ProcessUtils;
+
+public class TikaAsyncCLITest {
+ @Test
+ public void testCrash() throws Exception {
+ Path config = getPath("/tika-config-broken.xml");
+ assertThrows(TikaConfigException.class,
+ () -> TikaAsyncCLI.main(
+ new String[] {
+ ProcessUtils.escapeCommandLine(config.toAbsolutePath().toString())
+ })
+ );
+ }
+
+ private Path getPath(String file) throws Exception {
+ return Paths.get(this.getClass().getResource(file).toURI());
+ }
+}
diff --git a/tika-pipes/tika-async-cli/src/test/resources/tika-config-broken.xml b/tika-pipes/tika-async-cli/src/test/resources/tika-config-broken.xml
new file mode 100644
index 000000000..75b10da92
--- /dev/null
+++ b/tika-pipes/tika-async-cli/src/test/resources/tika-config-broken.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<properties>
+ <fetchers>
+ <fetcher class="org.apache.tika.pipes.fetcher.s3.S3Fetcher">
+ <params>
+ <name>s3</name>
+ <region>us-east-1</region>
+ <profile><!-- fill in here --></profile>
+ </params>
+ </fetcher>
+ </fetchers>
+ <pipesIterator class="org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator">
+ <params>
+ <fetcherName>fs</fetcherName>
+ <basePath>basePath</basePath>
+ </params>
+ </pipesIterator>
+</properties>
\ No newline at end of file