You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by nd...@apache.org on 2025/10/01 12:54:09 UTC
(tika) branch TIKA-4334 created (now 7c1efa31f)
This is an automated email from the ASF dual-hosted git repository.
ndipiazza pushed a change to branch TIKA-4334
in repository https://gitbox.apache.org/repos/asf/tika.git
at 7c1efa31f TIKA-4334: move grpc out of pipes and into tika-server, implement more controller
This branch includes the following new commits:
new 1c933becb TIKA-4334: replace pipes and server with newer apis
new af31f994f TIKA-4334: replace pipes and server with newer apis
new e46345a43 TIKA-4334: replace pipes and server with newer apis - add eval and detector controllers
new 7c1efa31f TIKA-4334: move grpc out of pipes and into tika-server, implement more controller
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(tika) 01/04: TIKA-4334: replace pipes and server with newer apis
Posted by nd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ndipiazza pushed a commit to branch TIKA-4334
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 1c933becbcb06a2a22a323389456aefad43957bf
Author: Nicholas DiPiazza <nd...@apache.org>
AuthorDate: Sun Sep 28 14:25:42 2025 -0500
TIKA-4334: replace pipes and server with newer apis
---
pom.xml | 2 +-
tika-grpc/README.md | 13 -
tika-grpc/pom.xml | 438 ----
.../tika/pipes/grpc/ExpiringFetcherStore.java | 101 -
.../org/apache/tika/pipes/grpc/TikaGrpcServer.java | 205 --
.../apache/tika/pipes/grpc/TikaGrpcServerImpl.java | 420 ----
tika-grpc/src/main/proto/tika.proto | 145 --
.../tika/pipes/grpc/ExpiringFetcherStoreTest.java | 65 -
...PipesBiDirectionalStreamingIntegrationTest.java | 217 --
.../apache/tika/pipes/grpc/TikaGrpcServerTest.java | 287 ---
tika-grpc/src/test/resources/certs/README.md | 5 -
tika-grpc/src/test/resources/certs/ca.key | 28 -
tika-grpc/src/test/resources/certs/ca.pem | 20 -
tika-grpc/src/test/resources/certs/client.key | 28 -
tika-grpc/src/test/resources/certs/client.pem | 20 -
tika-grpc/src/test/resources/certs/server1.key | 28 -
tika-grpc/src/test/resources/certs/server1.pem | 22 -
.../src/test/resources/tika-pipes-test-config.xml | 35 -
tika-parent/pom.xml | 20 +
tika-pipes/pom.xml | 99 +-
tika-pipes/tika-async-cli/pom.xml | 134 --
.../apache/tika/async/cli/SimpleAsyncConfig.java | 60 -
.../org/apache/tika/async/cli/TikaAsyncCLI.java | 159 --
.../tika/async/cli/TikaConfigAsyncWriter.java | 159 --
.../apache/tika/async/cli/AsyncCliParserTest.java | 82 -
.../apache/tika/async/cli/AsyncProcessorTest.java | 145 --
.../apache/tika/async/cli/TikaAsyncCLITest.java | 41 -
.../test/resources/configs/TIKA-4207-emitter.xml | 35 -
.../test/resources/configs/tika-config-broken.xml | 32 -
.../resources/test-documents/basic_embedded.xml | 35 -
tika-pipes/tika-emitters/pom.xml | 47 -
.../tika-emitters/tika-emitter-az-blob/pom.xml | 114 --
.../tika/pipes/emitter/azblob/AZBlobEmitter.java | 265 ---
.../pipes/emitter/azblob/TestAZBlobEmitter.java | 56 -
.../test/resources/config/tika-config-az-blob.xml | 28 -
tika-pipes/tika-emitters/tika-emitter-gcs/pom.xml | 114 --
.../apache/tika/pipes/emitter/gcs/GCSEmitter.java | 195 --
.../tika/pipes/emitter/gcs/TestGCSEmitter.java | 55 -
.../src/test/resources/config/tika-config-gcs.xml | 26 -
tika-pipes/tika-emitters/tika-emitter-jdbc/pom.xml | 59 -
.../tika/pipes/emitter/jdbc/JDBCEmitter.java | 701 -------
.../tika/pipes/emitter/jdbc/JDBCEmitterTest.java | 271 ---
.../tika-config-jdbc-emitter-attachments.xml | 53 -
.../tika-config-jdbc-emitter-existing-table.xml | 42 -
.../tika-config-jdbc-emitter-multivalued.xml | 45 -
.../configs/tika-config-jdbc-emitter-trunc.xml | 44 -
.../resources/configs/tika-config-jdbc-emitter.xml | 54 -
.../tika-emitters/tika-emitter-kafka/pom.xml | 118 --
.../tika/pipes/emitter/kafka/KafkaEmitter.java | 288 ---
.../tika-emitters/tika-emitter-opensearch/pom.xml | 135 --
.../pipes/emitter/opensearch/JsonResponse.java | 60 -
.../pipes/emitter/opensearch/OpenSearchClient.java | 336 ----
.../emitter/opensearch/OpenSearchEmitter.java | 241 ---
.../emitter/opensearch/OpenSearchClientTest.java | 52 -
.../test/resources/tika-config-simple-emitter.xml | 41 -
tika-pipes/tika-emitters/tika-emitter-s3/pom.xml | 112 --
.../apache/tika/pipes/emitter/s3/S3Emitter.java | 335 ----
tika-pipes/tika-emitters/tika-emitter-solr/pom.xml | 117 --
.../tika/pipes/emitter/solr/SolrEmitter.java | 360 ----
.../pipes/emitter/solr/SolrEmitterDevTest.java | 63 -
.../src/test/resources/log4j2.xml | 35 -
.../test/resources/tika-config-simple-emitter.xml | 48 -
tika-pipes/tika-fetchers/pom.xml | 48 -
.../tika-fetchers/tika-fetcher-az-blob/pom.xml | 119 --
.../tika/pipes/fetcher/azblob/AZBlobFetcher.java | 206 --
.../fetcher/azblob/config/AZBlobFetcherConfig.java | 72 -
.../pipes/fetcher/azblob/TestAZBlobFetcher.java | 55 -
.../src/test/resources/tika-config-az-blob.xml | 30 -
tika-pipes/tika-fetchers/tika-fetcher-gcs/pom.xml | 106 -
.../apache/tika/pipes/fetcher/gcs/GCSFetcher.java | 145 --
.../pipes/fetcher/gcs/config/GCSFetcherConfig.java | 62 -
.../tika/pipes/fetcher/s3/TestGCSFetcher.java | 63 -
.../src/test/resources/tika-config-gcs.xml | 26 -
tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml | 139 --
.../tika/pipes/fetcher/http/HttpFetcher.java | 690 -------
.../pipes/fetcher/http/config/HttpHeaders.java | 69 -
.../tika/pipes/fetcher/http/HttpFetcherTest.java | 301 ---
.../pipes/fetcher/http/config/HttpHeadersTest.java | 42 -
.../pipes/fetcher/http/jwt/JwtGeneratorTest.java | 56 -
.../src/test/resources/tika-config-http.xml | 32 -
.../tika-fetcher-microsoft-graph/pom.xml | 200 --
.../microsoftgraph/MicrosoftGraphFetcher.java | 140 --
.../config/AadCredentialConfigBase.java | 27 -
.../Client2CertificateCredentialsConfig.java | 50 -
.../config/ClientCertificateCredentialsConfig.java | 64 -
.../config/ClientSecretCredentialsConfig.java | 54 -
.../microsoftgraph/MicrosoftGraphFetcherTest.java | 105 -
tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml | 111 --
.../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 416 ----
.../pipes/fetcher/s3/config/S3FetcherConfig.java | 172 --
.../tika/pipes/fetcher/s3/TestS3Fetcher.java | 64 -
.../src/test/resources/tika-config-s3.xml | 27 -
tika-pipes/tika-httpclient-commons/pom.xml | 64 -
.../org/apache/tika/client/HttpClientFactory.java | 474 -----
.../org/apache/tika/client/HttpClientUtil.java | 76 -
.../apache/tika/client/TikaClientException.java | 29 -
tika-pipes/tika-pipes-cli/pom.xml | 112 ++
.../pipes/cli/filesystem/FileSystemFetcherCli.java | 163 ++
.../cli/googledrive/GoogleDriveFetcherCli.java | 143 ++
.../apache/tika/pipes/cli/http/HttpFetcherCli.java | 130 ++
.../pipes/cli/mapper/ObjectMapperProvider.java | 11 +
.../microsoftgraph/MicrosoftGraphFetcherCli.java | 172 ++
.../org/apache/tika/pipes/cli/s3/S3FetcherCli.java | 232 +++
.../tika-pipes-cli}/src/main/resources/log4j2.xml | 0
tika-pipes/tika-pipes-core/pom.xml | 136 +-
.../tika/pipes/core/CompositePipesReporter.java | 119 --
.../pipes/core/FailedToStartClientException.java | 27 -
.../org/apache/tika/pipes/core/FetchEmitTuple.java | 126 --
.../org/apache/tika/pipes/core/HandlerConfig.java | 172 --
.../org/apache/tika/pipes/core/PipesClient.java | 568 ------
.../org/apache/tika/pipes/core/PipesConfig.java | 66 -
.../apache/tika/pipes/core/PipesConfigBase.java | 192 --
.../org/apache/tika/pipes/core/PipesException.java | 27 -
.../org/apache/tika/pipes/core/PipesParser.java | 75 -
.../org/apache/tika/pipes/core/PipesReporter.java | 98 -
.../apache/tika/pipes/core/PipesReporterBase.java | 155 --
.../org/apache/tika/pipes/core/PipesResult.java | 114 --
.../org/apache/tika/pipes/core/PipesServer.java | 894 ---------
.../apache/tika/pipes/core/async/AsyncConfig.java | 120 --
.../apache/tika/pipes/core/async/AsyncEmitter.java | 144 --
.../tika/pipes/core/async/AsyncProcessor.java | 337 ----
.../apache/tika/pipes/core/async/AsyncStatus.java | 90 -
.../pipes/core/async/OfferLargerThanQueueSize.java | 41 -
.../tika/pipes/core/emitter/AbstractEmitter.java | 52 -
.../apache/tika/pipes/core/emitter/EmitData.java | 97 -
.../apache/tika/pipes/core/emitter/EmitKey.java | 80 -
.../apache/tika/pipes/core/emitter/Emitter.java | 35 -
.../tika/pipes/core/emitter/EmitterManager.java | 98 -
.../tika/pipes/core/emitter/StreamEmitter.java | 28 -
.../pipes/core/emitter/TikaEmitterException.java | 29 -
.../pipes/core/emitter/fs/FileSystemEmitter.java | 173 --
.../pipes/core/exception/TikaPipesException.java | 22 +
.../core/exception/TikaServerParseException.java | 22 +
.../AbstractEmbeddedDocumentBytesHandler.java | 78 -
.../BasicEmbeddedDocumentBytesHandler.java | 58 -
.../extractor/EmbeddedDocumentBytesConfig.java | 178 --
.../EmittingEmbeddedDocumentBytesHandler.java | 77 -
.../apache/tika/pipes/core/fetcher/FetchKey.java | 97 -
.../apache/tika/pipes/core/fetcher/Fetcher.java | 38 -
.../tika/pipes/core/fetcher/FetcherManager.java | 96 -
.../pipes/core/fetcher/FetcherStringException.java | 29 -
.../tika/pipes/core/fetcher/RangeFetcher.java | 40 -
.../pipes/core/fetcher/config/AbstractConfig.java | 21 -
.../fetcher/config/FetcherConfigContainer.java | 40 -
.../pipes/core/fetcher/fs/FileSystemFetcher.java | 182 --
.../fetcher/fs/config/FileSystemFetcherConfig.java | 42 -
.../tika/pipes/core/fetcher/url/UrlFetcher.java | 53 -
.../tika/pipes/core/parser/ParseService.java | 125 ++
.../core/pipesiterator/CallablePipesIterator.java | 125 --
.../pipes/core/pipesiterator/PipesIterator.java | 274 ---
.../pipes/core/pipesiterator/TotalCountResult.java | 54 -
.../pipes/core/pipesiterator/TotalCounter.java | 44 -
.../filelist/FileListPipesIterator.java | 107 -
.../pipesiterator/fs/FileSystemPipesIterator.java | 256 ---
.../pipes/core/serialization/JsonEmitData.java | 59 -
.../core/serialization/JsonFetchEmitTuple.java | 195 --
.../core/serialization/JsonFetchEmitTupleList.java | 75 -
.../apache/tika/pipes/core/util/CharsetUtil.java | 35 +
.../org/apache/tika/pipes/core/util/OSUtils.java | 258 +++
.../org/apache/tika/pipes/core/util/Props.java | 28 +
.../resources/pipes-fork-server-default-log4j2.xml | 32 -
.../apache/tika/pipes/core/PipesClientTest.java | 97 -
.../apache/tika/pipes/core/PipesServerTest.java | 192 --
.../tika/pipes/core/TikaPipesConfigTest.java | 123 --
.../pipes/core/async/AsyncChaosMonkeyTest.java | 202 --
.../tika/pipes/core/async/MockDigesterFactory.java | 56 -
.../apache/tika/pipes/core/async/MockEmitter.java | 59 -
.../apache/tika/pipes/core/async/MockFetcher.java | 44 -
.../apache/tika/pipes/core/async/MockReporter.java | 60 -
.../tika/pipes/core/async/MockReporterTest.java | 52 -
.../tika/pipes/core/emitter/MockEmitter.java | 60 -
.../tika/pipes/core/fetcher/MockFetcher.java | 72 -
.../core/fetcher/fs/FileSystemFetcherTest.java | 57 -
.../pipesiterator/FileSystemPipesIteratorTest.java | 81 -
.../filelist/FileListPipesIteratorTest.java | 79 -
.../serialization/JsonFetchEmitTupleListTest.java | 60 -
.../core/serialization/JsonFetchEmitTupleTest.java | 103 -
.../test/resources/configs/TIKA-4207-emitter.xml | 35 -
.../test/resources/configs/tika-config-broken.xml | 32 -
.../org/apache/tika/config/TIKA-3865-params.xml | 41 -
.../org/apache/tika/config/emitters-config.xml | 27 -
.../tika/config/emitters-duplicate-config.xml | 31 -
.../tika/config/emitters-mock-throw-on-check.xml | 27 -
.../org/apache/tika/config/fetchers-config.xml | 29 -
.../tika/config/fetchers-duplicate-config.xml | 33 -
.../tika/config/fetchers-mock-throw-on-check.xml | 27 -
.../tika/config/fetchers-nobasepath-config.xml | 28 -
.../apache/tika/config/fetchers-noname-config.xml | 32 -
.../apache/tika/config/pipes-iterator-config.xml | 25 -
.../tika/config/pipes-iterator-multiple-config.xml | 31 -
.../org/apache/tika/pipes/core/TIKA-3941.xml | 30 -
.../tika/pipes/core/TIKA-4207-limit-bytes.xml | 34 -
.../org/apache/tika/pipes/core/TIKA-4207.xml | 30 -
.../org/apache/tika/pipes/core/async/TIKA-3507.xml | 30 -
.../org/apache/tika/pipes/core/async/TIKA-3865.xml | 35 -
.../apache/tika/pipes/core/tika-sample-config.xml | 41 -
.../resources/test-documents/basic_embedded.xml | 35 -
.../test/resources/test-documents/file-list.txt | 10 -
.../resources/test-documents/mock/embedded.xml | 53 -
.../test/resources/test-documents/mock_times.xml | 26 -
.../test-documents/testOverlappingText.pdf | Bin 899 -> 0 bytes
tika-pipes/tika-pipes-emitters/README.md | 6 +
tika-pipes/tika-pipes-emitters/pom.xml | 109 +
.../tika-emitter-file-system/pom.xml | 82 +
.../src/main/assembly/assembly.xml | 30 +
.../fetchers/filesystem/FileSystemEmitter.java | 84 +
.../filesystem/FileSystemEmitterConfig.java | 45 +
.../filesystem/FileSystemEmitterPlugin.java} | 23 +-
.../src/main/resources/plugin.properties} | 13 +-
.../tika-pipes-emitters/tika-emitters-core/pom.xml | 38 +
.../pipes/core/emitter/DefaultEmitterConfig.java | 39 +
.../apache/tika/pipes/core/emitter/EmitOutput.java | 17 +
.../apache/tika/pipes/core/emitter/Emitter.java | 12 +
.../tika/pipes/core/emitter/EmitterConfig.java | 12 +
.../tika/pipes/core/emitter/OnExistBehavior.java | 5 +
.../pipes/core/emitter/TikaEmitterException.java | 11 +
tika-pipes/tika-pipes-fetchers/README.md | 53 +
tika-pipes/tika-pipes-fetchers/pom.xml | 114 ++
.../tika-fetcher-atlassian-jwt/pom.xml | 117 ++
.../src/main/assembly/assembly.xml | 30 +
.../fetchers/atlassianjwt/AtlassianJwtFetcher.java | 347 ++++
.../atlassianjwt/AtlassianJwtFetcherPlugin.java} | 30 +-
.../atlassianjwt/AtlassianJwtGenerator.java | 189 ++
.../config/AtlassianJwtFetcherConfig.java | 191 ++
.../src/main/resources/plugin.properties | 6 +
.../tika-fetcher-file-system/pom.xml | 82 +
.../src/main/assembly/assembly.xml | 30 +
.../fetchers/filesystem/FileSystemFetcher.java | 100 +
.../filesystem/FileSystemFetcherConfig.java | 29 +
.../filesystem/FileSystemFetcherPlugin.java} | 29 +-
.../src/main/resources/plugin.properties | 6 +
.../tika-fetcher-google-drive/pom.xml | 132 ++
.../src/main/assembly/assembly.xml | 30 +
.../fetchers/googledrive/GoogleDriveFetcher.java | 122 ++
.../fetchers/googledrive/GoogleDrivePlugin.java} | 27 +-
.../config/GoogleDriveFetcherConfig.java | 56 +
.../src/main/resources/plugin.properties | 6 +
.../src/test/resources/log4j2.xml | 0
.../tika-pipes-fetchers/tika-fetcher-http/pom.xml | 117 ++
.../src/main/assembly/assembly.xml | 30 +
.../tika/pipes/fetchers/http/HttpFetcher.java | 406 ++++
.../pipes/fetchers/http/HttpFetcherPlugin.java} | 27 +-
.../pipes/fetchers/http/config/AuthConfig.java | 54 +
.../fetchers}/http/config/HttpFetcherConfig.java | 152 +-
.../tika/pipes/fetchers/http/config/JwtConfig.java | 54 +
.../pipes/fetchers/http/config/ProxyConfig.java | 24 +
.../tika/pipes/fetchers}/http/jwt/JwtCreds.java | 2 +-
.../pipes/fetchers}/http/jwt/JwtGenerator.java | 2 +-
.../fetchers}/http/jwt/JwtPrivateKeyCreds.java | 2 +-
.../pipes/fetchers}/http/jwt/JwtSecretCreds.java | 2 +-
.../src/main/resources/plugin.properties | 6 +
.../tika-fetcher-microsoft-graph/pom.xml | 197 ++
.../src/main/assembly/assembly.xml | 30 +
.../microsoftgraph/MicrosoftGraphFetcher.java | 128 ++
.../microsoftgraph/MicrosoftGraphPlugin.java} | 27 +-
.../config/MicrosoftGraphFetcherConfig.java | 70 +-
.../src/main/resources/plugin.properties | 6 +
.../src/test/resources/log4j2.xml | 0
.../tika-pipes-fetchers/tika-fetcher-s3/pom.xml | 127 ++
.../tika-fetcher-s3/src/main/assembly/assembly.xml | 30 +
.../tika/pipes/fetchers/s3/S3ClientManager.java | 88 +
.../apache/tika/pipes/fetchers/s3/S3Fetcher.java | 181 ++
.../apache/tika/pipes/fetchers/s3/S3Plugin.java} | 29 +-
.../pipes/fetchers/s3/config/S3FetcherConfig.java | 67 +
.../src/main/resources/plugin.properties | 6 +
.../tika-fetcher-s3}/src/test/resources/log4j2.xml | 0
.../tika-pipes-fetchers/tika-fetchers-core/pom.xml | 38 +
.../pipes/fetchers/core/DefaultFetcherConfig.java | 39 +
.../apache/tika/pipes/fetchers/core/Fetcher.java | 10 +
.../tika/pipes/fetchers/core/FetcherConfig.java | 14 +
tika-pipes/tika-pipes-grpc/README.md | 17 +
.../tika-pipes-grpc/corpa-files}/014760.docx | Bin
.../tika-pipes-grpc/corpa-files}/017091.docx | Bin
.../tika-pipes-grpc/corpa-files}/017097.docx | Bin
.../tika-pipes-grpc/corpa-files}/018367.docx | Bin
tika-pipes/tika-pipes-grpc/corpa-files/018370.docx | Bin 0 -> 30723 bytes
tika-pipes/tika-pipes-grpc/corpa-files/018371.docx | Bin 0 -> 26036 bytes
tika-pipes/tika-pipes-grpc/corpa-files/021524.docx | Bin 0 -> 16028 bytes
.../corpa-files/file-that-confuses-tika.log | 419 ++++
tika-pipes/tika-pipes-grpc/docker-build/Dockerfile | 40 +
.../tika-pipes-grpc/docker-build/docker-build.sh | 119 ++
.../docker-build/start-tika-grpc.sh | 33 +
tika-pipes/tika-pipes-grpc/pom.xml | 388 ++++
.../apache/tika/pipes/TikaPipesApplication.java | 12 +
.../tika/pipes/grpc/TikaGrpcServerCustomizer.java | 19 +
.../apache/tika/pipes/grpc/TikaGrpcService.java | 754 +++++++
.../java/org/apache/tika/pipes/job/JobStatus.java | 19 +
.../tika/pipes/model/FetchAndParseStatus.java | 6 +
.../parser/TikaPipesApplicationConfiguration.java | 49 +
.../plugin/ClasspathPluginPropertiesFinder.java | 18 +
.../tika/pipes/plugin/GrpcPluginManager.java | 100 +
.../org/apache/tika/pipes/plugin/PluginConfig.java | 28 +
.../apache/tika/pipes/repo/EmitterRepository.java | 19 +
.../apache/tika/pipes/repo/FetcherRepository.java | 19 +
.../pipes/repo/IgniteRepositoryConfiguration.java | 69 +
.../tika/pipes/repo/JobStatusRepository.java | 18 +
.../tika/pipes/repo/PipeIteratorRepository.java | 19 +
.../src/main/resources/application.yaml | 25 +
.../tika/pipes/TikaPipesIntegrationTestBase.java | 72 +
.../pipes/grpc/TikaGrpcServerEmittersCrudTest.java | 79 +
.../grpc/TikaGrpcServerFetchAndParseTest.java | 166 ++
.../pipes/grpc/TikaGrpcServerFetchersCrudTest.java | 77 +
.../pipes/grpc/TikaGrpcServerJsonSchemaTest.java | 77 +
.../grpc/TikaGrpcServerPipeIteratorsCrudTest.java | 81 +
.../tika/pipes/grpc/TikaGrpcServerPipeJobTest.java | 156 ++
tika-pipes/tika-pipes-iterators/pom.xml | 50 -
.../tika-pipes-iterator-az-blob/pom.xml | 108 -
.../pipesiterator/azblob/AZBlobPipesIterator.java | 151 --
.../azblob/TestAZBlobPipesIterator.java | 99 -
.../tika-pipes-iterator-csv/pom.xml | 118 --
.../pipes/pipesiterator/csv/CSVPipesIterator.java | 254 ---
.../src/test/java/TestCSVPipesIterator.java | 131 --
.../src/test/resources/test-simple.csv | 6 -
.../tika-pipes-iterator-gcs/pom.xml | 108 -
.../pipes/pipesiterator/gcs/GCSPipesIterator.java | 126 --
.../pipesiterator/gcs/TestGCSPipesIterator.java | 101 -
.../tika-pipes-iterator-jdbc/pom.xml | 109 -
.../pipesiterator/jdbc/JDBCPipesIterator.java | 379 ----
.../pipesiterator/jdbc/TestJDBCPipesIterator.java | 188 --
.../src/test/resources/log4j2.xml | 32 -
.../tika-pipes-iterator-json/pom.xml | 120 --
.../pipesiterator/json/JsonPipesIterator.java | 65 -
.../pipesiterator/json/TestJsonPipesIterator.java | 92 -
.../test-documents/test-with-embedded-bytes.json | 2100 --------------------
.../src/test/resources/test-documents/test.json | 1400 -------------
.../tika-pipes-iterator-kafka/pom.xml | 117 --
.../pipesiterator/kafka/KafkaPipesIterator.java | 174 --
.../kafka/TestKafkaPipesIterator.java | 99 -
.../src/test/resources/log4j2.xml | 32 -
.../tika-pipes-iterator-s3/pom.xml | 113 --
.../pipes/pipesiterator/s3/S3PipesIterator.java | 217 --
.../pipesiterator/s3/TestS3PipesIterator.java | 100 -
.../src/test/resources/log4j2.xml | 32 -
.../tika-pipes-iterator-solr/pom.xml | 114 --
.../pipesiterator/solr/SolrPipesIterator.java | 277 ---
tika-pipes/tika-pipes-pipe-iterators/README.md | 11 +
tika-pipes/tika-pipes-pipe-iterators/pom.xml | 109 +
.../tika-pipe-iterator-csv/pom.xml | 94 +
.../src/main/assembly/assembly.xml | 30 +
.../pipes/fetchers/filesystem/CsvPipeIterator.java | 90 +
.../fetchers/filesystem/CsvPipeIteratorConfig.java | 21 +
.../filesystem/CsvPipeIteratorPlugin.java} | 32 +-
.../src/main/resources/plugin.properties | 6 +
.../tika-pipe-iterators-core/pom.xml | 39 +
.../core/iterators/DefaultPipeIteratorConfig.java | 44 +
.../tika/pipes/core/iterators/PipeInput.java | 15 +
.../tika/pipes/core/iterators/PipeIterator.java | 12 +
.../pipes/core/iterators/PipeIteratorConfig.java | 14 +
.../core/iterators/TikaPipeIteratorException.java | 11 +
tika-pipes/tika-pipes-proto/pom.xml | 184 ++
.../src/main/proto/google/api/annotations.proto | 31 +
.../src/main/proto/google/api/http.proto | 371 ++++
.../tika-pipes-proto/src/main/proto/tika.proto | 440 ++++
tika-pipes/tika-pipes-reporters/pom.xml | 43 -
.../tika-pipes-reporter-fs-status/pom.xml | 135 --
.../reporters/fs/FileSystemStatusReporter.java | 199 --
.../reporters/fs/TestFileSystemStatusReporter.java | 181 --
.../tika-pipes-reporter-jdbc/pom.xml | 108 -
.../pipes/reporters/jdbc/JDBCPipesReporter.java | 490 -----
.../reporters/jdbc/TestJDBCPipesReporter.java | 297 ---
.../resources/configs/tika-config-advanced.xml | 50 -
.../resources/configs/tika-config-excludes.xml | 42 -
.../resources/configs/tika-config-includes.xml | 42 -
.../tika-pipes-reporter-opensearch/pom.xml | 113 --
.../pipes/reporters/opensearch/JsonResponse.java | 60 -
.../reporters/opensearch/OpenSearchClient.java | 168 --
.../opensearch/OpenSearchPipesReporter.java | 243 ---
{tika-server => tika-server-old}/README.md | 0
{tika-server => tika-server-old}/pom.xml | 21 +
.../tika-server-client/pom.xml | 2 +
.../tika/server/client/TikaAsyncHttpClient.java | 0
.../org/apache/tika/server/client/TikaClient.java | 0
.../apache/tika/server/client/TikaClientCLI.java | 0
.../server/client/TikaClientConfigException.java | 0
.../tika/server/client/TikaEmitterResult.java | 0
.../tika/server/client/TikaPipesHttpClient.java | 0
.../tika/server/client/TikaServerClientConfig.java | 0
.../org/apache/tika/server/client/TestBasic.java | 0
.../resources/tika-config-simple-fs-emitter.xml | 0
.../tika-server-core/pom.xml | 2 +
.../server/core/CompositeParseContextConfig.java | 0
.../server/core/DefaultInputStreamFactory.java | 0
.../tika/server/core/FetcherStreamFactory.java | 0
.../org/apache/tika/server/core/HTMLHelper.java | 0
.../tika/server/core/InputStreamFactory.java | 0
.../org/apache/tika/server/core/MetadataList.java | 0
.../tika/server/core/ParseContextConfig.java | 0
.../server/core/ProduceTypeResourceComparator.java | 0
.../org/apache/tika/server/core/ServerStatus.java | 0
.../tika/server/core/ServerStatusResource.java | 0
.../tika/server/core/ServerStatusWatcher.java | 0
.../org/apache/tika/server/core/TaskStatus.java | 0
.../apache/tika/server/core/TikaLoggingFilter.java | 0
.../org/apache/tika/server/core/TikaServerCli.java | 0
.../apache/tika/server/core/TikaServerConfig.java | 0
.../tika/server/core/TikaServerParseException.java | 0
.../core/TikaServerParseExceptionMapper.java | 0
.../apache/tika/server/core/TikaServerProcess.java | 0
.../tika/server/core/TikaServerWatchDog.java | 0
.../org/apache/tika/server/core/TlsConfig.java | 0
.../apache/tika/server/core/WatchDogResult.java | 0
.../server/core/config/DocumentSelectorConfig.java | 0
.../server/core/config/PasswordProviderConfig.java | 0
.../tika/server/core/config/TimeoutConfig.java | 0
.../tika/server/core/resource/AsyncRequest.java | 0
.../tika/server/core/resource/AsyncResource.java | 0
.../server/core/resource/DetectorResource.java | 0
.../server/core/resource/LanguageResource.java | 0
.../server/core/resource/MetadataResource.java | 0
.../tika/server/core/resource/PipesResource.java | 0
.../core/resource/RecursiveMetadataResource.java | 0
.../tika/server/core/resource/TikaDetectors.java | 0
.../tika/server/core/resource/TikaMimeTypes.java | 0
.../tika/server/core/resource/TikaParsers.java | 0
.../tika/server/core/resource/TikaResource.java | 0
.../server/core/resource/TikaServerResource.java | 0
.../server/core/resource/TikaServerStatus.java | 0
.../tika/server/core/resource/TikaVersion.java | 0
.../tika/server/core/resource/TikaWelcome.java | 0
.../server/core/resource/TranslateResource.java | 0
.../server/core/resource/UnpackerResource.java | 0
.../server/core/writer/CSVMessageBodyWriter.java | 0
.../server/core/writer/JSONMessageBodyWriter.java | 0
.../tika/server/core/writer/JSONObjWriter.java | 0
.../core/writer/MetadataListMessageBodyWriter.java | 0
.../apache/tika/server/core/writer/TarWriter.java | 0
.../server/core/writer/TextMessageBodyWriter.java | 0
.../tika/server/core/writer/TikaServerWriter.java | 0
.../apache/tika/server/core/writer/ZipWriter.java | 0
.../org.apache.tika.server.core.ParseContextConfig | 0
.../main/resources/tika-server-config-default.xml | 0
.../src/main/resources/tikaserver-template.html | 0
.../main/resources/tikaserver-version.properties | 0
.../org/apache/tika/server/core/CXFTestBase.java | 0
.../tika/server/core/IntegrationTestBase.java | 0
.../tika/server/core/LanguageResourceTest.java | 0
.../tika/server/core/NullWebClientLogger.java | 0
.../server/core/RecursiveMetadataResourceTest.java | 0
.../apache/tika/server/core/ServerStatusTest.java | 0
.../apache/tika/server/core/StackTraceOffTest.java | 0
.../apache/tika/server/core/StackTraceTest.java | 0
.../apache/tika/server/core/TikaMimeTypesTest.java | 0
.../org/apache/tika/server/core/TikaPipesTest.java | 0
.../tika/server/core/TikaResourceFetcherTest.java | 0
.../core/TikaResourceMetadataFilterTest.java | 0
.../tika/server/core/TikaResourceNoStackTest.java | 0
.../apache/tika/server/core/TikaResourceTest.java | 0
.../core/TikaServerAsyncIntegrationTest.java | 0
.../tika/server/core/TikaServerConfigTest.java | 0
.../server/core/TikaServerIntegrationTest.java | 0
.../core/TikaServerPipesIntegrationTest.java | 0
.../tika/server/core/TikaServerStatusTest.java | 0
.../apache/tika/server/core/TikaVersionTest.java | 0
.../apache/tika/server/core/TikaWelcomeTest.java | 0
.../tika/server/core/TranslateResourceTest.java | 0
.../resources/configs/metadata-filter-include.xml | 0
.../configs/tika-config-server-badjvmargs.xml | 0
.../resources/configs/tika-config-server-basic.xml | 0
.../configs/tika-config-server-emitter.xml | 0
.../tika-config-server-fetcher-template.xml | 0
.../tika-config-server-fetchers-emitters.xml | 0
.../configs/tika-config-server-timeout-10000.xml | 0
.../tika-config-server-tls-one-way-template.xml | 0
.../tika-config-server-tls-two-way-template.xml | 0
.../resources/configs/tika-config-server-tls.xml | 0
.../test/resources/configs/tika-config-server.xml | 0
.../src/test/resources/logging/log4j2_forked.xml | 0
.../src/test/resources/ssl-keys/README.txt | 0
.../resources/ssl-keys/tika-client-keystore.p12 | Bin
.../resources/ssl-keys/tika-client-truststore.p12 | Bin
.../src/test/resources/ssl-keys/tika-client.crt | Bin
.../resources/ssl-keys/tika-server-keystore.p12 | Bin
.../resources/ssl-keys/tika-server-truststore.p12 | Bin
.../src/test/resources/ssl-keys/tika-server.crt | Bin
.../src/test/resources/test-documents/english.txt | 0
.../src/test/resources/test-documents/french.txt | 0
.../mock/encrypted_document_exception.xml | 0
.../resources/test-documents/mock/fake_oom.xml | 0
.../test-documents/mock/heavy_hang_100.xml | 0
.../test-documents/mock/heavy_hang_30000.xml | 0
.../resources/test-documents/mock/hello_world.xml | 0
.../test-documents/mock/hello_world_long.xml | 0
.../resources/test-documents/mock/null_pointer.xml | 0
.../resources/test-documents/mock/real_oom.xml | 0
.../resources/test-documents/mock/system_exit.xml | 0
.../test-documents/mock/testStaticStdOutErr.xml | 0
.../test-documents/mock/testStdOutErr.xml | 0
.../test-documents/mock/thread_interrupt.xml | 0
.../mock/\344\270\255\346\226\207.xml" | 0
.../test/resources/test-documents/testDigilite.fdf | 0
.../tika-server-eval/pom.xml | 0
.../apache/tika/server/eval/TikaEvalResource.java | 0
...he.tika.server.core.resource.TikaServerResource | 0
.../tika/server/eval/TikaEvalResourceTest.java | 0
tika-server-old/tika-server-spring/pom.xml | 170 ++
.../apache/tika/server/TikaServerApplication.java | 55 +
.../tika/server/controller/AsyncController.java | 182 ++
.../tika/server/controller/DetectorController.java | 84 +
.../controller/InformationServicesController.java | 282 +++
.../controller/LanguageResourceController.java | 142 ++
.../tika/server/controller/MetadataController.java | 177 ++
.../tika/server/controller/PipesController.java | 194 ++
.../RecursiveMetadataAndContentController.java | 240 +++
.../server/controller/ServerStatusController.java | 101 +
.../tika/server/controller/TikaController.java | 292 +++
.../controller/TranslateResourceController.java | 118 ++
.../controller/UnpackResourceController.java | 249 +++
.../main/resources/api/tika-server-openapi.yaml | 1098 ++++++++++
.../src/main/resources/application.properties | 30 +-
.../tika-server-standard/assembly.xml | 0
.../tika-server-standard/bin/init.d/tika | 0
.../bin/install_tika_service.sh | 0
.../tika-server-standard/bin/tika | 0
.../tika-server-standard/bin/tika.in.sh | 0
.../tika-server-standard/pom.xml | 0
.../server/standard/config/PDFServerConfig.java | 0
.../standard/config/TesseractServerConfig.java | 0
.../standard/resource/XMPMetadataResource.java | 0
.../standard/writer/XMPMessageBodyWriter.java | 0
.../services/org.apache.tika.parser.Parser | 0
.../org.apache.tika.server.core.ParseContextConfig | 0
...he.tika.server.core.resource.TikaServerResource | 0
...apache.tika.server.core.writer.TikaServerWriter | 0
.../tika/server/standard/DetectorResourceTest.java | 0
.../apache/tika/server/standard/FetcherTest.java | 0
.../server/standard/JsonMaxFieldLengthTest.java | 0
.../tika/server/standard/MetadataResourceTest.java | 0
.../server/standard/OpenNLPMetadataFilterTest.java | 0
.../standard/OptimaizeMetadataFilterTest.java | 0
.../standard/RecursiveMetadataFilterTest.java | 0
.../standard/RecursiveMetadataResourceTest.java | 0
.../tika/server/standard/TikaDetectorsTest.java | 0
.../tika/server/standard/TikaMimeTypesTest.java | 0
.../tika/server/standard/TikaParsersTest.java | 0
.../apache/tika/server/standard/TikaPipesTest.java | 0
.../tika/server/standard/TikaResourceTest.java | 0
.../tika/server/standard/UnpackerResourceTest.java | 0
.../standard/UnpackerResourceWithConfigTest.java | 0
.../test/resources/config/TIKA-3137-include.xml | 0
.../config/tika-config-for-server-tests.xml | 0
.../src/test/resources/config/tika-config-json.xml | 0
.../tika-config-langdetect-opennlp-filter.xml | 0
.../tika-config-langdetect-optimaize-filter.xml | 0
.../test/resources/config/tika-config-unpacker.xml | 0
.../resources/config/tika-config-url-fetcher.xml | 0
.../src/test/resources/test-documents/2exe.docx | Bin
.../src/test/resources/test-documents/2pic.doc | Bin
.../src/test/resources/test-documents/2pic.docx | Bin
.../test-documents/CDEC_WEATHER_2010_03_02 | 0
.../src/test/resources/test-documents/Doc1_ole.doc | Bin
.../src/test/resources/test-documents/foo.csv | 0
.../src/test/resources/test-documents/password.xls | Bin
.../src/test/resources/test-documents/pic.xls | Bin
.../src/test/resources/test-documents/pic.xlsx | Bin
.../src/test/resources/test-documents/test.doc | Bin
.../test-documents/testColorRendering.pdf | Bin
.../test/resources/test-documents/testHTML.html | 0
.../src/test/resources/test-documents/testOCR.pdf | Bin
.../test-documents/testPDFTwoTextBoxes.pdf | Bin
.../test-documents/testPDF_XFA_govdocs1_258578.pdf | Bin
.../test-documents/testPassword4Spaces.pdf | Bin
.../testRTF_npeFromWMFInTikaServer.rtf | 0
.../test-documents/testUnicodePassword.pdf | Bin
.../test-documents/test_recursive_embedded.docx | Bin
.../src/main/resources/log4j2.xml | 32 -
.../src/test/resources/log4j2.xml | 32 -
.../tika-server-core/src/main/resources/log4j2.xml | 32 -
.../tika-server-core/src/test/resources/log4j2.xml | 32 -
568 files changed, 13433 insertions(+), 29825 deletions(-)
diff --git a/pom.xml b/pom.xml
index 58635e421..be20e234d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,7 @@
<module>tika-xmp</module>
<module>tika-langdetect</module>
<module>tika-pipes</module>
- <module>tika-grpc</module>
+ <module>tika-server-spring</module>
<module>tika-app</module>
<module>tika-server</module>
<module>tika-integration-tests</module>
diff --git a/tika-grpc/README.md b/tika-grpc/README.md
deleted file mode 100644
index f2269b22b..000000000
--- a/tika-grpc/README.md
+++ /dev/null
@@ -1,13 +0,0 @@
-# Tika Pipes GRPC Server
-
-The following is the Tika Pipes GRPC Server.
-
-This server will manage a pool of Tika Pipes clients.
-
-* Tika Pipes Fetcher CRUD operations
- * Create
- * Read
- * Update
- * Delete
-* Fetch + Parse a given Fetch Item
-
diff --git a/tika-grpc/pom.xml b/tika-grpc/pom.xml
deleted file mode 100644
index cf3433da3..000000000
--- a/tika-grpc/pom.xml
+++ /dev/null
@@ -1,438 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>tika-grpc</artifactId>
- <packaging>jar</packaging>
- <!-- Feel free to delete the comment at the end of these lines. It is just
- for safely updating the version in our release process. -->
- <name>Apache Tika pipes gRPC server</name>
- <url>https://tika.apache.org/</url>
-
- <parent>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-parent</artifactId>
- <version>4.0.0-SNAPSHOT</version>
- <relativePath>../tika-parent/pom.xml</relativePath>
- </parent>
-
- <properties>
- <grpc.version>1.75.0</grpc.version>
- <asarkar-grpc-test.version>2.0.0</asarkar-grpc-test.version>
- <awaitility.version>4.3.0</awaitility.version>
- <j2objc-annotations.version>3.1</j2objc-annotations.version>
- </properties>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-bom</artifactId>
- <version>${grpc.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
- <dependencies>
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-netty-shaded</artifactId>
- <scope>runtime</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.errorprone</groupId>
- <artifactId>error_prone_annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.errorprone</groupId>
- <artifactId>error_prone_annotations</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-services</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.errorprone</groupId>
- <artifactId>error_prone_annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.j2objc</groupId>
- <artifactId>j2objc-annotations</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-stub</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.errorprone</groupId>
- <artifactId>error_prone_annotations</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java-util</artifactId>
- <version>${protobuf.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.errorprone</groupId>
- <artifactId>error_prone_annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.j2objc</groupId>
- <artifactId>j2objc-annotations</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.google.j2objc</groupId>
- <artifactId>j2objc-annotations</artifactId>
- <version>${j2objc-annotations.version}</version> <!-- prevent downgrade of version in guava -->
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-async-cli</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-parsers-standard-package</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.errorprone</groupId>
- <artifactId>error_prone_annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.j2objc</groupId>
- <artifactId>j2objc-annotations</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <!-- logging -->
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.tomcat</groupId>
- <artifactId>annotations-api</artifactId>
- <version>6.0.53</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-testing</artifactId>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-fetcher-http</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-jsonSchema</artifactId>
- </dependency>
- <dependency>
- <groupId>com.asarkar.grpc</groupId>
- <artifactId>grpc-test</artifactId>
- <version>${asarkar-grpc-test.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.awaitility</groupId>
- <artifactId>awaitility</artifactId>
- <version>${awaitility.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <extensions>
- <extension>
- <groupId>kr.motd.maven</groupId>
- <artifactId>os-maven-plugin</artifactId>
- <version>1.7.1</version>
- </extension>
- </extensions>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>${maven.javadoc.version}</version>
- <configuration>
- <doclint>none</doclint>
- <sourcepath>src/main/java;${project.build.directory}/generated-sources/protobuf/grpc-java:${project.build.directory}/generated-sources/protobuf/java</sourcepath>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.xolstice.maven.plugins</groupId>
- <artifactId>protobuf-maven-plugin</artifactId>
- <version>0.6.1</version>
- <configuration>
- <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
- <pluginId>grpc-java</pluginId>
- <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>compile</goal>
- <goal>compile-custom</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-enforcer-plugin</artifactId>
- <executions>
- <execution>
- <id>enforce</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <configuration>
- <rules>
- <requireUpperBoundDeps />
- </rules>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes>
- <exclude>src/test/resources/certs/**</exclude>
- <exclude>README.md</exclude>
- </excludes>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>3.6.1</version>
- <executions>
- <execution>
- <id>test</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>${basedir}/target/generated-sources/protobuf/grpc-java</source>
- <source>${basedir}/target/generated-sources/protobuf/java</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>3.5.1</version>
- <executions>
- <execution>
- <goals>
- <goal>java</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <mainClass>org.apache.tika.pipes.grpc.TikaGrpcServer</mainClass>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <artifactSet>
- <excludes>
- </excludes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>module-info.class</exclude>
- <exclude>META-INF/maven/plugin.xml</exclude>
- <exclude>META-INF/versions/9/module-info.class</exclude>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- <exclude>META-INF/*.txt</exclude>
- <exclude>META-INF/ASL2.0</exclude>
- <exclude>META-INF/DEPENDENCIES</exclude>
- <exclude>META-INF/LICENSE</exclude>
- <exclude>META-INF/NOTICE</exclude>
- <exclude>META-INF/README</exclude>
- <exclude>META-INF/MANIFEST.MF</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- <exclude>CHANGES</exclude>
- <exclude>README</exclude>
- <exclude>builddef.lst</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.tika.pipes.grpc.TikaGrpcServer</mainClass>
- <manifestEntries>
- <Multi-Release>true</Multi-Release>
- </manifestEntries>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <version>${checkstyle.plugin.version}</version>
- <dependencies>
- <dependency>
- <groupId>com.puppycrawl.tools</groupId>
- <artifactId>checkstyle</artifactId>
- <version>${puppycrawl.version}</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>validate</id>
- <phase>validate</phase>
- <configuration>
- <configLocation>checkstyle.xml</configLocation>
- <inputEncoding>UTF-8</inputEncoding>
- <consoleOutput>true</consoleOutput>
- <logViolationsToConsole>true</logViolationsToConsole>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <testSourceDirectories>${project.basedir}/src/test/java</testSourceDirectories>
- <violationSeverity>error</violationSeverity>
- <failOnViolation>true</failOnViolation>
- <excludeGeneratedSources>true</excludeGeneratedSources>
- </configuration>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/ExpiringFetcherStore.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/ExpiringFetcherStore.java
deleted file mode 100644
index 3dc10f09c..000000000
--- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/ExpiringFetcherStore.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.grpc;
-
-import java.time.Instant;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
-
-public class ExpiringFetcherStore implements AutoCloseable {
- private static final Logger LOG = LoggerFactory.getLogger(ExpiringFetcherStore.class);
- public static final long EXPIRE_JOB_INITIAL_DELAY = 1L;
- private final Map<String, AbstractFetcher> fetchers = Collections.synchronizedMap(new HashMap<>());
- private final Map<String, AbstractConfig> fetcherConfigs = Collections.synchronizedMap(new HashMap<>());
- private final Map<String, Instant> fetcherLastAccessed = Collections.synchronizedMap(new HashMap<>());
-
- private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
-
- public ExpiringFetcherStore(int expireAfterSeconds, int checkForExpiredFetchersDelaySeconds) {
- executorService.scheduleAtFixedRate(() -> {
- Set<String> expired = new HashSet<>();
- for (String fetcherName : fetchers.keySet()) {
- Instant lastAccessed = fetcherLastAccessed.get(fetcherName);
- if (lastAccessed == null) {
- LOG.error("Detected a fetcher with no last access time. FetcherName={}", fetcherName);
- expired.add(fetcherName);
- } else if (Instant
- .now()
- .isAfter(lastAccessed.plusSeconds(expireAfterSeconds))) {
- LOG.info("Detected stale fetcher {} hasn't been accessed in {} seconds. " + "Deleting.", fetcherName, Instant
- .now()
- .getEpochSecond() - lastAccessed.getEpochSecond());
- expired.add(fetcherName);
- }
- }
- for (String expiredFetcherId : expired) {
- deleteFetcher(expiredFetcherId);
- }
- }, EXPIRE_JOB_INITIAL_DELAY, checkForExpiredFetchersDelaySeconds, TimeUnit.SECONDS);
- }
-
- public boolean deleteFetcher(String fetcherName) {
- boolean success = fetchers.remove(fetcherName) != null;
- fetcherConfigs.remove(fetcherName);
- fetcherLastAccessed.remove(fetcherName);
- return success;
- }
-
- public Map<String, AbstractFetcher> getFetchers() {
- return fetchers;
- }
-
- public Map<String, AbstractConfig> getFetcherConfigs() {
- return fetcherConfigs;
- }
-
- /**
- * This method will get the fetcher, but will also log the access the fetcher as having
- * been accessed. This prevents the scheduled job from removing the stale fetcher.
- */
- public <T extends AbstractFetcher> T getFetcherAndLogAccess(String fetcherName) {
- fetcherLastAccessed.put(fetcherName, Instant.now());
- return (T) fetchers.get(fetcherName);
- }
-
- public <T extends AbstractFetcher, C extends AbstractConfig> void createFetcher(T fetcher, C config) {
- fetchers.put(fetcher.getName(), fetcher);
- fetcherConfigs.put(fetcher.getName(), config);
- getFetcherAndLogAccess(fetcher.getName());
- }
-
- @Override
- public void close() {
- executorService.shutdownNow();
- }
-}
diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServer.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServer.java
deleted file mode 100644
index c9be5fb33..000000000
--- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServer.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.grpc;
-
-import static io.grpc.health.v1.HealthCheckResponse.ServingStatus;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.nio.charset.StandardCharsets;
-import java.util.concurrent.TimeUnit;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import io.grpc.Grpc;
-import io.grpc.InsecureServerCredentials;
-import io.grpc.Server;
-import io.grpc.ServerCredentials;
-import io.grpc.TlsServerCredentials;
-import io.grpc.protobuf.services.HealthStatusManager;
-import io.grpc.protobuf.services.ProtoReflectionServiceV1;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.config.TikaConfigSerializer;
-
-/**
- * Server that manages startup/shutdown of the GRPC Tika server.
- */
-public class TikaGrpcServer {
- private static final Logger LOGGER = LoggerFactory.getLogger(TikaGrpcServer.class);
- public static final int TIKA_SERVER_GRPC_DEFAULT_PORT = 50052;
- private Server server;
- @Parameter(names = {"-p", "--port"}, description = "The grpc server port", help = true)
- private Integer port = TIKA_SERVER_GRPC_DEFAULT_PORT;
-
- @Parameter(names = {"-c", "--config"}, description = "The grpc server port", help = true)
- private File tikaConfigXml;
-
- @Parameter(names = {"-s", "--secure"}, description = "Enable credentials required to access this grpc server")
- private boolean secure;
-
- @Parameter(names = {"--cert-chain"}, description = "Certificate chain file. Example: server1.pem See: https://github.com/grpc/grpc-java/tree/b3ffb5078df361d7460786e134db7b5c00939246/examples/example-tls")
- private File certChain;
-
- @Parameter(names = {"--private-key"}, description = "Private key store. Example: server1.key See: https://github.com/grpc/grpc-java/tree/b3ffb5078df361d7460786e134db7b5c00939246/examples/example-tls")
- private File privateKey;
-
- @Parameter(names = {"--private-key-password"}, description = "Private key password, if needed")
- private String privateKeyPassword;
-
- @Parameter(names = {"--trust-cert-collection"}, description = "The trust certificate collection (root certs). Example: ca.pem See: https://github.com/grpc/grpc-java/tree/b3ffb5078df361d7460786e134db7b5c00939246/examples/example-tls")
- private File trustCertCollection;
-
- @Parameter(names = {"--client-auth-required"}, description = "Is Mutual TLS required?")
- private boolean clientAuthRequired;
-
- @Parameter(names = {"-h", "-H", "--help"}, description = "Display help menu")
- private boolean help;
-
- public void start() throws Exception {
- HealthStatusManager healthStatusManager = new HealthStatusManager();
- ServerCredentials creds;
- if (secure) {
- TlsServerCredentials.Builder channelCredBuilder = TlsServerCredentials.newBuilder();
- channelCredBuilder.keyManager(certChain, privateKey, privateKeyPassword);
- if (trustCertCollection != null && trustCertCollection.exists()) {
- channelCredBuilder.trustManager(trustCertCollection);
- if (clientAuthRequired) {
- channelCredBuilder.clientAuth(TlsServerCredentials.ClientAuth.REQUIRE);
- }
- }
- creds = channelCredBuilder.build();
- } else {
- creds = InsecureServerCredentials.create();
- }
- if (tikaConfigXml == null) {
- // Create a default tika config
- tikaConfigXml = File.createTempFile("tika-config", ".xml");
- try (FileWriter fw = new FileWriter(tikaConfigXml, StandardCharsets.UTF_8)) {
- TikaConfigSerializer.serialize(new TikaConfig(), TikaConfigSerializer.Mode.STATIC_FULL, fw, StandardCharsets.UTF_8);
- }
- }
- File tikaConfigFile = new File(tikaConfigXml.getAbsolutePath());
- healthStatusManager.setStatus(TikaGrpcServer.class.getSimpleName(), ServingStatus.SERVING);
- server = Grpc
- .newServerBuilderForPort(port, creds)
- .addService(new TikaGrpcServerImpl(tikaConfigFile.getAbsolutePath()))
- .addService(healthStatusManager.getHealthService())
- .addService(ProtoReflectionServiceV1.newInstance())
- .build()
- .start();
- LOGGER.info("Server started, listening on " + port);
- Runtime
- .getRuntime()
- .addShutdownHook(new Thread(() -> {
- // Use stderr here since the logger may have been reset by its JVM shutdown hook.
- System.err.println("*** shutting down gRPC server since JVM is shutting down");
- healthStatusManager.clearStatus(TikaGrpcServer.class.getSimpleName());
- try {
- TikaGrpcServer.this.stop();
- } catch (InterruptedException e) {
- e.printStackTrace(System.err);
- }
- System.err.println("*** server shut down");
- }));
- }
-
- public void stop() throws InterruptedException {
- if (server != null) {
- server
- .shutdown()
- .awaitTermination(30, TimeUnit.SECONDS);
- }
- }
-
- /**
- * Await termination on the main thread since the grpc library uses daemon threads.
- */
- public void blockUntilShutdown() throws InterruptedException {
- if (server != null) {
- server.awaitTermination();
- }
- }
-
- /**
- * Main launches the server from the command line.
- */
- public static void main(String[] args) throws Exception {
- TikaGrpcServer server = new TikaGrpcServer();
- JCommander commander = JCommander
- .newBuilder()
- .addObject(server)
- .build();
-
- commander.parse(args);
-
- if (server.help) {
- commander.usage();
- return;
- }
-
- server.start();
- server.blockUntilShutdown();
- }
-
- public TikaGrpcServer setTikaConfigXml(File tikaConfigXml) {
- this.tikaConfigXml = tikaConfigXml;
- return this;
- }
-
- public TikaGrpcServer setServer(Server server) {
- this.server = server;
- return this;
- }
-
- public TikaGrpcServer setPort(Integer port) {
- this.port = port;
- return this;
- }
-
- public TikaGrpcServer setSecure(boolean secure) {
- this.secure = secure;
- return this;
- }
-
- public TikaGrpcServer setCertChain(File certChain) {
- this.certChain = certChain;
- return this;
- }
-
- public TikaGrpcServer setPrivateKey(File privateKey) {
- this.privateKey = privateKey;
- return this;
- }
-
- public TikaGrpcServer setPrivateKeyPassword(String privateKeyPassword) {
- this.privateKeyPassword = privateKeyPassword;
- return this;
- }
-
- public TikaGrpcServer setTrustCertCollection(File trustCertCollection) {
- this.trustCertCollection = trustCertCollection;
- return this;
- }
-
- public TikaGrpcServer setClientAuthRequired(boolean clientAuthRequired) {
- this.clientAuthRequired = clientAuthRequired;
- return this;
- }
-}
diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java
deleted file mode 100644
index 7f06cb681..000000000
--- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java
+++ /dev/null
@@ -1,420 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.grpc;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.module.jsonSchema.JsonSchema;
-import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator;
-import com.google.rpc.Status;
-import io.grpc.protobuf.StatusProto;
-import io.grpc.stub.StreamObserver;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.DeleteFetcherReply;
-import org.apache.tika.DeleteFetcherRequest;
-import org.apache.tika.FetchAndParseReply;
-import org.apache.tika.FetchAndParseRequest;
-import org.apache.tika.GetFetcherConfigJsonSchemaReply;
-import org.apache.tika.GetFetcherConfigJsonSchemaRequest;
-import org.apache.tika.GetFetcherReply;
-import org.apache.tika.GetFetcherRequest;
-import org.apache.tika.ListFetchersReply;
-import org.apache.tika.ListFetchersRequest;
-import org.apache.tika.SaveFetcherReply;
-import org.apache.tika.SaveFetcherRequest;
-import org.apache.tika.TikaGrpc;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.PipesClient;
-import org.apache.tika.pipes.core.PipesConfig;
-import org.apache.tika.pipes.core.PipesResult;
-import org.apache.tika.pipes.core.emitter.EmitKey;
-import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
-import org.apache.tika.pipes.core.fetcher.FetchKey;
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
-import org.apache.tika.pipes.core.fetcher.config.FetcherConfigContainer;
-import org.apache.tika.utils.XMLReaderUtils;
-
-class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase {
- private static final Logger LOG = LoggerFactory.getLogger(TikaGrpcServerImpl.class);
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- static {
- OBJECT_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL);
- }
- public static final JsonSchemaGenerator JSON_SCHEMA_GENERATOR = new JsonSchemaGenerator(OBJECT_MAPPER);
-
- /**
- * FetcherID is key, The pair is the Fetcher object and the Metadata
- */
- PipesConfig pipesConfig;
- PipesClient pipesClient;
- ExpiringFetcherStore expiringFetcherStore;
-
- String tikaConfigPath;
-
- TikaGrpcServerImpl(String tikaConfigPath)
- throws TikaConfigException, IOException, ParserConfigurationException,
- TransformerException, SAXException {
- File tikaConfigFile = new File(tikaConfigPath);
- if (!tikaConfigFile.canWrite()) {
- File tmpTikaConfigFile = File.createTempFile("configCopy", tikaConfigFile.getName());
- tmpTikaConfigFile.deleteOnExit();
- LOG.info("Tika config file {} is read-only. Making a temporary copy to {}", tikaConfigFile, tmpTikaConfigFile);
- String tikaConfigFileContents = FileUtils.readFileToString(tikaConfigFile, StandardCharsets.UTF_8);
- FileUtils.writeStringToFile(tmpTikaConfigFile, tikaConfigFileContents, StandardCharsets.UTF_8);
- tikaConfigFile = tmpTikaConfigFile;
- tikaConfigPath = tikaConfigFile.getAbsolutePath();
- }
- pipesConfig = PipesConfig.load(tikaConfigFile.toPath());
- pipesClient = new PipesClient(pipesConfig);
-
- expiringFetcherStore = new ExpiringFetcherStore(pipesConfig.getStaleFetcherTimeoutSeconds(),
- pipesConfig.getStaleFetcherDelaySeconds());
- this.tikaConfigPath = tikaConfigPath;
- try {
- updateTikaConfig();
- } catch (TikaException e) {
- throw new TikaConfigException("Problem updating tikaConfig", e);
- }
- }
-
- private void updateTikaConfig() throws ParserConfigurationException, IOException, SAXException, TransformerException, TikaException {
- Document tikaConfigDoc =
- DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(tikaConfigPath);
-
- Element fetchersElement = (Element) tikaConfigDoc.getElementsByTagName("fetchers").item(0);
- if (fetchersElement == null) {
- fetchersElement = tikaConfigDoc.createElement("fetchers");
- tikaConfigDoc.getDocumentElement().appendChild(fetchersElement);
- }
- for (int i = 0; i < fetchersElement.getChildNodes().getLength(); ++i) {
- fetchersElement.removeChild(fetchersElement.getChildNodes().item(i));
- }
- for (var fetcherEntry : expiringFetcherStore.getFetchers().entrySet()) {
- AbstractFetcher fetcherObject = fetcherEntry.getValue();
- Map<String, Object> fetcherConfigParams = OBJECT_MAPPER.convertValue(
- expiringFetcherStore.getFetcherConfigs().get(fetcherEntry.getKey()),
- new TypeReference<>() {
- });
- Element fetcher = tikaConfigDoc.createElement("fetcher");
- fetcher.setAttribute("class", fetcherEntry.getValue().getClass().getName());
- Element fetcherName = tikaConfigDoc.createElement("name");
- fetcherName.setTextContent(fetcherObject.getName());
- fetcher.appendChild(fetcherName);
- populateFetcherConfigs(fetcherConfigParams, tikaConfigDoc, fetcher);
- fetchersElement.appendChild(fetcher);
- }
- DOMSource source = new DOMSource(tikaConfigDoc);
- FileWriter writer = new FileWriter(tikaConfigPath, StandardCharsets.UTF_8);
- StreamResult result = new StreamResult(writer);
-
- TransformerFactory transformerFactory = XMLReaderUtils.getTransformerFactory();
- Transformer transformer = transformerFactory.newTransformer();
- transformer.transform(source, result);
- }
-
- private void populateFetcherConfigs(Map<String, Object> fetcherConfigParams,
- Document tikaConfigDoc, Element fetcher) {
- for (var configParam : fetcherConfigParams.entrySet()) {
- Element configElm = tikaConfigDoc.createElement(configParam.getKey());
- fetcher.appendChild(configElm);
- if (configParam.getValue() instanceof List) {
- List configParamVal = (List) configParam.getValue();
- String singularName = configParam.getKey().substring(0, configParam.getKey().length() - 1);
- for (Object configParamObj : configParamVal) {
- Element childElement = tikaConfigDoc.createElement(singularName);
- childElement.setTextContent(Objects.toString(configParamObj));
- configElm.appendChild(childElement);
- }
- } else {
- configElm.setTextContent(Objects.toString(configParam.getValue()));
- }
- }
- }
-
- @Override
- public void fetchAndParseServerSideStreaming(FetchAndParseRequest request,
- StreamObserver<FetchAndParseReply> responseObserver) {
- fetchAndParseImpl(request, responseObserver);
- }
-
- @Override
- public StreamObserver<FetchAndParseRequest> fetchAndParseBiDirectionalStreaming(
- StreamObserver<FetchAndParseReply> responseObserver) {
- return new StreamObserver<>() {
- @Override
- public void onNext(FetchAndParseRequest fetchAndParseRequest) {
- fetchAndParseImpl(fetchAndParseRequest, responseObserver);
- }
-
- @Override
- public void onError(Throwable throwable) {
- LOG.error("Parse error occurred", throwable);
- }
-
- @Override
- public void onCompleted() {
- responseObserver.onCompleted();
- }
- };
- }
-
- @Override
- public void fetchAndParse(FetchAndParseRequest request,
- StreamObserver<FetchAndParseReply> responseObserver) {
- fetchAndParseImpl(request, responseObserver);
- responseObserver.onCompleted();
- }
-
-
- private void fetchAndParseImpl(FetchAndParseRequest request,
- StreamObserver<FetchAndParseReply> responseObserver) {
- AbstractFetcher fetcher =
- expiringFetcherStore.getFetcherAndLogAccess(request.getFetcherId());
- if (fetcher == null) {
- throw new RuntimeException(
- "Could not find fetcher with name " + request.getFetcherId());
- }
- Metadata tikaMetadata = new Metadata();
- try {
- ParseContext parseContext = new ParseContext();
- String additionalFetchConfigJson = request.getAdditionalFetchConfigJson();
- if (StringUtils.isNotBlank(additionalFetchConfigJson)) {
- // The fetch and parse has the option to specify additional configuration
- AbstractConfig abstractConfig = expiringFetcherStore
- .getFetcherConfigs()
- .get(fetcher.getName());
- parseContext.set(FetcherConfigContainer.class, new FetcherConfigContainer()
- .setConfigClassName(abstractConfig
- .getClass().getName())
- .setJson(additionalFetchConfigJson));
- }
- PipesResult pipesResult = pipesClient.process(new FetchEmitTuple(request.getFetchKey(),
- new FetchKey(fetcher.getName(), request.getFetchKey()), new EmitKey(), tikaMetadata, parseContext, FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP));
- FetchAndParseReply.Builder fetchReplyBuilder =
- FetchAndParseReply.newBuilder()
- .setFetchKey(request.getFetchKey())
- .setStatus(pipesResult.getStatus().name());
- if (pipesResult.getStatus().equals(PipesResult.STATUS.FETCH_EXCEPTION)) {
- fetchReplyBuilder.setErrorMessage(pipesResult.getMessage());
- }
- if (pipesResult.getEmitData() != null && pipesResult.getEmitData().getMetadataList() != null) {
- for (Metadata metadata : pipesResult.getEmitData().getMetadataList()) {
- for (String name : metadata.names()) {
- String value = metadata.get(name);
- if (value != null) {
- fetchReplyBuilder.putFields(name, value);
- }
- }
- }
- }
- responseObserver.onNext(fetchReplyBuilder.build());
- } catch (IOException e) {
- throw new RuntimeException(e);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
-
- @SuppressWarnings("raw")
- @Override
- public void saveFetcher(SaveFetcherRequest request,
- StreamObserver<SaveFetcherReply> responseObserver) {
- SaveFetcherReply reply =
- SaveFetcherReply.newBuilder().setFetcherId(request.getFetcherId()).build();
- try {
- Map<String, Object> fetcherConfigMap = OBJECT_MAPPER.readValue(request.getFetcherConfigJson(), new TypeReference<>() {});
- Map<String, Param> tikaParamsMap = createTikaParamMap(fetcherConfigMap);
- saveFetcher(request.getFetcherId(), request.getFetcherClass(), fetcherConfigMap, tikaParamsMap);
- updateTikaConfig();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- responseObserver.onNext(reply);
- responseObserver.onCompleted();
- }
-
- private void saveFetcher(String name, String fetcherClassName, Map<String, Object> paramsMap, Map<String, Param> tikaParamsMap) {
- try {
- if (paramsMap == null) {
- paramsMap = new LinkedHashMap<>();
- }
- Class<? extends AbstractFetcher> fetcherClass =
- (Class<? extends AbstractFetcher>) Class.forName(fetcherClassName);
- String configClassName =
- fetcherClass.getPackageName() + ".config." + fetcherClass.getSimpleName() +
- "Config";
- Class<? extends AbstractConfig> configClass =
- (Class<? extends AbstractConfig>) Class.forName(configClassName);
- AbstractConfig configObject = OBJECT_MAPPER.convertValue(paramsMap, configClass);
- AbstractFetcher abstractFetcher =
- fetcherClass.getDeclaredConstructor(configClass).newInstance(configObject);
- abstractFetcher.setName(name);
- if (Initializable.class.isAssignableFrom(fetcherClass)) {
- Initializable initializable = (Initializable) abstractFetcher;
- initializable.initialize(tikaParamsMap);
- }
- if (expiringFetcherStore.deleteFetcher(name)) {
- LOG.info("Updating fetcher {}", name);
- } else {
- LOG.info("Creating new fetcher {}", name);
- }
- expiringFetcherStore.createFetcher(abstractFetcher, configObject);
- } catch (ClassNotFoundException | InstantiationException | IllegalAccessException |
- InvocationTargetException | NoSuchMethodException | TikaConfigException e) {
- throw new RuntimeException(e);
- }
- }
-
- private static Map<String, Param> createTikaParamMap(Map<String, Object> fetcherConfigMap) {
- Map<String, Param> tikaParamsMap = new HashMap<>();
- for (Map.Entry<String, Object> entry : fetcherConfigMap.entrySet()) {
- if (entry.getValue() != null) {
- tikaParamsMap.put(entry.getKey(), new Param<>(entry.getKey(), entry.getValue()));
- }
- }
- return tikaParamsMap;
- }
-
- static Status notFoundStatus(String fetcherId) {
- return Status.newBuilder()
- .setCode(io.grpc.Status.Code.NOT_FOUND.value())
- .setMessage("Could not find fetcher with id:" + fetcherId)
- .build();
- }
-
- @Override
- public void getFetcher(GetFetcherRequest request,
- StreamObserver<GetFetcherReply> responseObserver) {
- GetFetcherReply.Builder getFetcherReply = GetFetcherReply.newBuilder();
- AbstractConfig abstractConfig =
- expiringFetcherStore.getFetcherConfigs().get(request.getFetcherId());
- AbstractFetcher abstractFetcher = expiringFetcherStore.getFetchers().get(request.getFetcherId());
- if (abstractFetcher == null || abstractConfig == null) {
- responseObserver.onError(StatusProto.toStatusException(notFoundStatus(request.getFetcherId())));
- return;
- }
- getFetcherReply.setFetcherId(request.getFetcherId());
- getFetcherReply.setFetcherClass(abstractFetcher.getClass().getName());
- Map<String, Object> paramMap = OBJECT_MAPPER.convertValue(abstractConfig, new TypeReference<>() {});
- paramMap.forEach(
- (k, v) -> getFetcherReply.putParams(Objects.toString(k), Objects.toString(v)));
- responseObserver.onNext(getFetcherReply.build());
- responseObserver.onCompleted();
- }
-
- @Override
- public void listFetchers(ListFetchersRequest request,
- StreamObserver<ListFetchersReply> responseObserver) {
- ListFetchersReply.Builder listFetchersReplyBuilder = ListFetchersReply.newBuilder();
- for (Map.Entry<String, AbstractConfig> fetcherConfig : expiringFetcherStore.getFetcherConfigs()
- .entrySet()) {
- GetFetcherReply.Builder replyBuilder = saveFetcherReply(fetcherConfig);
- listFetchersReplyBuilder.addGetFetcherReplies(replyBuilder.build());
- }
- responseObserver.onNext(listFetchersReplyBuilder.build());
- responseObserver.onCompleted();
- }
-
- private GetFetcherReply.Builder saveFetcherReply(
- Map.Entry<String, AbstractConfig> fetcherConfig) {
- AbstractFetcher abstractFetcher =
- expiringFetcherStore.getFetchers().get(fetcherConfig.getKey());
- AbstractConfig abstractConfig =
- expiringFetcherStore.getFetcherConfigs().get(fetcherConfig.getKey());
- GetFetcherReply.Builder replyBuilder =
- GetFetcherReply.newBuilder().setFetcherClass(abstractFetcher.getClass().getName())
- .setFetcherId(abstractFetcher.getName());
- loadParamsIntoReply(abstractConfig, replyBuilder);
- return replyBuilder;
- }
-
- private static void loadParamsIntoReply(AbstractConfig abstractConfig,
- GetFetcherReply.Builder replyBuilder) {
- Map<String, Object> paramMap =
- OBJECT_MAPPER.convertValue(abstractConfig, new TypeReference<>() {
- });
- if (paramMap != null) {
- paramMap.forEach(
- (k, v) -> replyBuilder.putParams(Objects.toString(k), Objects.toString(v)));
- }
- }
-
- @Override
- public void deleteFetcher(DeleteFetcherRequest request,
- StreamObserver<DeleteFetcherReply> responseObserver) {
- boolean successfulDelete = deleteFetcher(request.getFetcherId());
- if (successfulDelete) {
- try {
- updateTikaConfig();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- responseObserver.onNext(DeleteFetcherReply.newBuilder().setSuccess(successfulDelete).build());
- responseObserver.onCompleted();
- }
-
- @Override
- public void getFetcherConfigJsonSchema(GetFetcherConfigJsonSchemaRequest request, StreamObserver<GetFetcherConfigJsonSchemaReply> responseObserver) {
- GetFetcherConfigJsonSchemaReply.Builder builder = GetFetcherConfigJsonSchemaReply.newBuilder();
- try {
- JsonSchema jsonSchema = JSON_SCHEMA_GENERATOR.generateSchema(Class.forName(request.getFetcherClass()));
- builder.setFetcherConfigJsonSchema(OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(jsonSchema));
- } catch (ClassNotFoundException | JsonProcessingException e) {
- throw new RuntimeException("Could not create json schema for " + request.getFetcherClass(), e);
- }
- responseObserver.onNext(builder.build());
- responseObserver.onCompleted();
- }
-
- private boolean deleteFetcher(String fetcherName) {
- return expiringFetcherStore.deleteFetcher(fetcherName);
- }
-}
diff --git a/tika-grpc/src/main/proto/tika.proto b/tika-grpc/src/main/proto/tika.proto
deleted file mode 100644
index 572ded7ab..000000000
--- a/tika-grpc/src/main/proto/tika.proto
+++ /dev/null
@@ -1,145 +0,0 @@
-// Copyright 2015 The gRPC Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-syntax = "proto3";
-package tika;
-
-option java_multiple_files = true;
-option java_package = "org.apache.tika";
-option java_outer_classname = "TikaProto";
-option objc_class_prefix = "HLW";
-
-// The Tika Grpc Service definition
-service Tika {
- /*
- Save a fetcher to the fetcher store.
- */
- rpc SaveFetcher(SaveFetcherRequest) returns (SaveFetcherReply) {}
- /*
- Get a fetcher's data from the fetcher store.
- */
- rpc GetFetcher(GetFetcherRequest) returns (GetFetcherReply) {}
- /*
- List fetchers that are currently in the fetcher store.
- */
- rpc ListFetchers(ListFetchersRequest) returns (ListFetchersReply) {}
- /*
- Delete a fetcher from the fetcher store.
- */
- rpc DeleteFetcher(DeleteFetcherRequest) returns (DeleteFetcherReply) {}
- /*
- Using a Fetcher in the fetcher store, send a FetchAndParse request. This will fetch, parse, and return
- the FetchParseTuple data output from Tika Pipes. This is a synchronous call that immediately returns 1 result.
- */
- rpc FetchAndParse(FetchAndParseRequest) returns (FetchAndParseReply) {}
- /*
- Using a Fetcher in the fetcher store, send a FetchAndParse request. This will fetch, parse, and return
- the FetchParseTuple data output from Tika Pipes. This will stream the data from the server in response.
- */
- rpc FetchAndParseServerSideStreaming(FetchAndParseRequest)
- returns (stream FetchAndParseReply) {}
- /*
- Using a Fetcher in the fetcher store, send a FetchAndParse request. This will fetch, parse, and return
- the FetchParseTuple data output from Tika Pipes. This serves a bi-directional stream of fetch inputs and
- parsed outputs.
- */
- rpc FetchAndParseBiDirectionalStreaming(stream FetchAndParseRequest)
- returns (stream FetchAndParseReply) {}
- /*
- Get the Fetcher Config schema for a given fetcher class.
- */
- rpc GetFetcherConfigJsonSchema(GetFetcherConfigJsonSchemaRequest) returns (GetFetcherConfigJsonSchemaReply) {}
-}
-
-message SaveFetcherRequest {
- // A unique identifier for each fetcher. If this already exists, operation will overwrite existing.
- string fetcher_id = 1;
- // The full java class name of the fetcher class. List of
- // fetcher classes is found here: https://cwiki.apache.org/confluence/display/TIKA/tika-pipes
- string fetcher_class = 2;
- // JSON string of the fetcher config object. To see the json schema from which to build this json,
- // use the GetFetcherConfigJsonSchema rpc method.
- string fetcher_config_json = 3;
-}
-
-message SaveFetcherReply {
- // The fetcher_id that was saved.
- string fetcher_id = 1;
-}
-
-message FetchAndParseRequest {
- // The ID of the fetcher in the fetcher store (previously saved by SaveFetcher) to use for the fetch.
- string fetcher_id = 1;
- // The "Fetch Key" of the item that will be fetched.
- string fetch_key = 2;
- // You can supply additional fetch configuration using this. Follows same fetch configuration json schema
- // as the fetcher configuration.
- string additional_fetch_config_json = 3;
-}
-
-message FetchAndParseReply {
- // Echoes the fetch_key that was sent in the request.
- string fetch_key = 1;
- // Metadata fields from the parse output.
- map<string, string> fields = 2;
- // The status from the message. See javadoc for org.apache.tika.pipes.PipesResult.STATUS for the list of status.
- string status = 3;
- // If there was an error, this will contain the error message.
- string error_message = 4;
-}
-
-message DeleteFetcherRequest {
- // ID of the fetcher to delete.
- string fetcher_id = 1;
-}
-
-message DeleteFetcherReply {
- // Success if the fetcher was successfully removed from the fetch store.
- bool success = 1;
-}
-
-message GetFetcherRequest {
- // ID of the fetcher for which to return config.
- string fetcher_id = 1;
-}
-
-message GetFetcherReply {
- // Echoes the ID of the fetcher being returned.
- string fetcher_id = 1;
- // The full Java class name of the Fetcher.
- string fetcher_class = 2;
- // The configuration parameters.
- map<string, string> params = 3;
-}
-
-message ListFetchersRequest {
- // List the fetchers starting at this page number
- int32 page_number = 1;
- // List this many fetchers per page.
- int32 num_fetchers_per_page = 2;
-}
-
-message ListFetchersReply {
- // List of fetcher configs returned by the Lists Fetchers service.
- repeated GetFetcherReply get_fetcher_replies = 1;
-}
-
-message GetFetcherConfigJsonSchemaRequest {
- // The full java class name of the fetcher config for which to fetch json schema.
- string fetcher_class = 1;
-}
-
-message GetFetcherConfigJsonSchemaReply {
- // The json schema that describes the fetcher config in string format.
- string fetcher_config_json_schema = 1;
-}
diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/ExpiringFetcherStoreTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/ExpiringFetcherStoreTest.java
deleted file mode 100644
index 91a8b8a19..000000000
--- a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/ExpiringFetcherStoreTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.grpc;
-
-import static org.junit.jupiter.api.Assertions.assertNull;
-
-import java.io.InputStream;
-import java.time.Duration;
-
-import org.awaitility.Awaitility;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
-
-class ExpiringFetcherStoreTest {
-
- @Test
- void createFetcher() {
- try (ExpiringFetcherStore expiringFetcherStore = new ExpiringFetcherStore(1, 5)) {
- AbstractFetcher fetcher = new AbstractFetcher() {
- @Override
- public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) {
- return null;
- }
- };
- fetcher.setName("nick");
- AbstractConfig config = new AbstractConfig() {
- };
- expiringFetcherStore.createFetcher(fetcher, config);
-
- Assertions.assertNotNull(expiringFetcherStore
- .getFetchers()
- .get(fetcher.getName()));
-
- Awaitility
- .await()
- .atMost(Duration.ofSeconds(60))
- .until(() -> expiringFetcherStore
- .getFetchers()
- .get(fetcher.getName()) == null);
-
- assertNull(expiringFetcherStore
- .getFetcherConfigs()
- .get(fetcher.getName()));
- }
- }
-}
diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java
deleted file mode 100644
index c540e6100..000000000
--- a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.grpc;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.ServerSocket;
-import java.nio.file.Paths;
-import java.time.Duration;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.ImmutableMap;
-import io.grpc.Grpc;
-import io.grpc.ManagedChannel;
-import io.grpc.TlsChannelCredentials;
-import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;
-import io.grpc.stub.StreamObserver;
-import org.apache.commons.io.FileUtils;
-import org.awaitility.Awaitility;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.handler.ResourceHandler;
-import org.eclipse.jetty.util.resource.PathResource;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.FetchAndParseReply;
-import org.apache.tika.FetchAndParseRequest;
-import org.apache.tika.SaveFetcherReply;
-import org.apache.tika.SaveFetcherRequest;
-import org.apache.tika.TikaGrpc;
-import org.apache.tika.pipes.fetcher.http.HttpFetcher;
-
-/**
- * This test will start an HTTP server using jetty.
- * Then it will start Tika Pipes Grpc service.
- * Then it will, using a bidirectional stream of data, send urls to the
- * HTTP fetcher whilst simultaneously receiving parsed output as they parse.
- */
-class PipesBiDirectionalStreamingIntegrationTest {
- static final Logger LOGGER = LoggerFactory.getLogger(PipesBiDirectionalStreamingIntegrationTest.class);
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- static File tikaConfigXmlTemplate = Paths
- .get("src", "test", "resources", "tika-pipes-test-config.xml")
- .toFile();
- static File tikaConfigXml = new File("target", "tika-config-" + UUID.randomUUID() + ".xml");
- static TikaGrpcServer grpcServer;
- static int grpcPort;
- static String httpServerUrl;
- static TikaGrpc.TikaBlockingStub tikaBlockingStub;
- static TikaGrpc.TikaStub tikaStub;
- static Server httpServer;
- static int httpServerPort;
- String httpFetcherId = "httpFetcherIdHere";
- List<String> files = Arrays.asList("014760.docx", "017091.docx", "017097.docx", "018367.docx");
-
- static int findAvailablePort() throws IOException {
- try (ServerSocket serverSocket = new ServerSocket(0)) {
- return serverSocket.getLocalPort();
- }
- }
-
- @BeforeAll
- static void setUpHttpServer() throws Exception {
- // Specify the folder from which files will be served
- httpServerPort = findAvailablePort();
- httpServer = new Server(httpServerPort);
-
- ResourceHandler resourceHandler = new ResourceHandler();
- resourceHandler.setDirAllowed(true);
- // TODO when using jetty 12:
- // resourceHandler.setBaseResourceAsString("src/test/resources/test-files")
- resourceHandler.setBaseResource(new PathResource(Paths.get("src", "test", "resources", "test-files")));
- httpServer.setHandler(resourceHandler);
- httpServer.start();
-
- httpServerUrl = "http://" + InetAddress
- .getByName("localhost")
- .getHostAddress() + ":" + httpServerPort;
- }
-
- @BeforeAll
- static void setUpGrpcServer() throws Exception {
- grpcPort = findAvailablePort();
- FileUtils.copyFile(tikaConfigXmlTemplate, tikaConfigXml);
-
- grpcServer = new TikaGrpcServer();
- grpcServer.setTikaConfigXml(tikaConfigXml);
- grpcServer.setPort(grpcPort);
- grpcServer.setSecure(true);
- grpcServer.setCertChain(Paths.get("src", "test", "resources", "certs", "server1.pem").toFile());
- grpcServer.setPrivateKey(Paths.get("src", "test", "resources", "certs", "server1.key").toFile());
- grpcServer.setTrustCertCollection(Paths.get("src", "test", "resources", "certs", "ca.pem").toFile());
- grpcServer.setClientAuthRequired(true);
- grpcServer.start();
-
- String target = InetAddress
- .getByName("localhost")
- .getHostAddress() + ":" + grpcPort;
-
- TlsChannelCredentials.Builder channelCredBuilder = TlsChannelCredentials.newBuilder();
- File clientCertChain = Paths.get("src", "test", "resources", "certs", "client.pem").toFile();
- File clientPrivateKey = Paths.get("src", "test", "resources", "certs", "client.key").toFile();
- channelCredBuilder.keyManager(clientCertChain, clientPrivateKey);
- channelCredBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE.getTrustManagers());
-
- ManagedChannel channel = Grpc
- .newChannelBuilder(target, channelCredBuilder.build())
- .build();
-
- tikaBlockingStub = TikaGrpc.newBlockingStub(channel);
- tikaStub = TikaGrpc.newStub(channel);
- }
-
- @AfterAll
- static void stopHttpServer() throws Exception {
- if (httpServer != null) {
- httpServer.stop();
- }
- }
-
- @AfterAll
- static void stopGrpcServer() throws Exception {
- if (grpcServer != null) {
- grpcServer.stop();
- }
- }
-
- @AfterAll
- static void cleanConfig() throws Exception {
- FileUtils.deleteQuietly(tikaConfigXml);
- }
-
- @BeforeEach
- void createHttpFetcher() throws Exception {
- SaveFetcherRequest saveFetcherRequest = SaveFetcherRequest
- .newBuilder()
- .setFetcherId(httpFetcherId)
- .setFetcherClass(HttpFetcher.class.getName())
- .setFetcherConfigJson(OBJECT_MAPPER.writeValueAsString(ImmutableMap
- .builder()
- .put("requestTimeout", 30_000)
- .put("socketTimeout", 30_000)
- .put("connectTimeout", 20_000)
- .put("maxConnectionsPerRoute", 200)
- .put("maxRedirects", 0)
- .put("maxSpoolSize", -1)
- .put("overallTimeout", 50_000)
- .build()))
- .build();
- SaveFetcherReply saveFetcherReply = tikaBlockingStub.saveFetcher(saveFetcherRequest);
- Assertions.assertEquals(saveFetcherReply.getFetcherId(), httpFetcherId);
- }
-
- @Test
- void testHttpFetchScenario() throws Exception {
- AtomicInteger numParsed = new AtomicInteger();
- Map<String, Map<String, String>> result = Collections.synchronizedMap(new HashMap<>());
- StreamObserver<FetchAndParseReply> responseObserver = new StreamObserver<>() {
- @Override
- public void onNext(FetchAndParseReply fetchAndParseReply) {
- LOGGER.info("Parsed: {}", fetchAndParseReply.getFetchKey());
- numParsed.incrementAndGet();
- result.put(fetchAndParseReply.getFetchKey(), fetchAndParseReply.getFieldsMap());
- }
-
- @Override
- public void onError(Throwable throwable) {
- LOGGER.error("Error occurred", throwable);
- }
-
- @Override
- public void onCompleted() {
- LOGGER.info("Completed fetching.");
- }
- };
- StreamObserver<FetchAndParseRequest> request = tikaStub.fetchAndParseBiDirectionalStreaming(responseObserver);
- for (String file : files) {
- request.onNext(FetchAndParseRequest
- .newBuilder()
- .setFetcherId(httpFetcherId)
- .setFetchKey(httpServerUrl + "/" + file)
- .build());
- }
- request.onCompleted();
-
- Awaitility.await().atMost(Duration.ofSeconds(600)).until(() -> result.size() == files.size());
-
- Assertions.assertEquals(files.size(), numParsed.get());
- }
-}
diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java
deleted file mode 100644
index a17ce1634..000000000
--- a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.grpc;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
-
-import java.io.File;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Paths;
-import java.time.Duration;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
-import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import com.asarkar.grpc.test.GrpcCleanupExtension;
-import com.asarkar.grpc.test.Resources;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.ImmutableMap;
-import io.grpc.ManagedChannel;
-import io.grpc.Server;
-import io.grpc.Status;
-import io.grpc.StatusRuntimeException;
-import io.grpc.inprocess.InProcessChannelBuilder;
-import io.grpc.inprocess.InProcessServerBuilder;
-import io.grpc.stub.StreamObserver;
-import org.apache.commons.io.FileUtils;
-import org.jetbrains.annotations.NotNull;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.DeleteFetcherReply;
-import org.apache.tika.DeleteFetcherRequest;
-import org.apache.tika.FetchAndParseReply;
-import org.apache.tika.FetchAndParseRequest;
-import org.apache.tika.GetFetcherReply;
-import org.apache.tika.GetFetcherRequest;
-import org.apache.tika.SaveFetcherReply;
-import org.apache.tika.SaveFetcherRequest;
-import org.apache.tika.TikaGrpc;
-import org.apache.tika.pipes.core.PipesResult;
-import org.apache.tika.pipes.core.fetcher.fs.FileSystemFetcher;
-
-@ExtendWith(GrpcCleanupExtension.class)
-public class TikaGrpcServerTest {
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- private static final Logger LOG = LoggerFactory.getLogger(TikaGrpcServerTest.class);
- public static final int NUM_TEST_DOCS = 2;
- static File tikaConfigXmlTemplate = Paths
- .get("src", "test", "resources", "tika-pipes-test-config.xml")
- .toFile();
- static File tikaConfigXml = new File("target", "tika-config-" + UUID.randomUUID() + ".xml");
-
-
- @BeforeAll
- static void init() throws Exception {
- FileUtils.copyFile(tikaConfigXmlTemplate, tikaConfigXml);
- }
-
- @AfterAll
- static void clean() {
- tikaConfigXml.setWritable(true);
- FileUtils.deleteQuietly(tikaConfigXml);
- }
-
- static final int NUM_FETCHERS_TO_CREATE = 10;
-
- @Test
- public void testFetcherCrud(Resources resources) throws Exception {
- Assertions.assertTrue(tikaConfigXml.setWritable(false));
- String serverName = InProcessServerBuilder.generateName();
-
- Server server = InProcessServerBuilder
- .forName(serverName)
- .directExecutor()
- .addService(new TikaGrpcServerImpl(tikaConfigXml.getAbsolutePath()))
- .build()
- .start();
- resources.register(server, Duration.ofSeconds(10));
-
- ManagedChannel channel = InProcessChannelBuilder
- .forName(serverName)
- .directExecutor()
- .build();
- resources.register(channel, Duration.ofSeconds(10));
- TikaGrpc.TikaBlockingStub blockingStub = TikaGrpc.newBlockingStub(channel);
-
- String targetFolder = new File("target").getAbsolutePath();
- // create fetchers
- for (int i = 0; i < NUM_FETCHERS_TO_CREATE; ++i) {
- String fetcherId = createFetcherId(i);
- SaveFetcherReply reply = blockingStub.saveFetcher(SaveFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .setFetcherClass(FileSystemFetcher.class.getName())
- .setFetcherConfigJson(OBJECT_MAPPER.writeValueAsString(ImmutableMap
- .builder()
- .put("basePath", targetFolder)
- .put("extractFileSystemMetadata", true)
- .build()))
- .build());
- assertEquals(fetcherId, reply.getFetcherId());
- }
- // update fetchers
- for (int i = 0; i < NUM_FETCHERS_TO_CREATE; ++i) {
- String fetcherId = createFetcherId(i);
- SaveFetcherReply reply = blockingStub.saveFetcher(SaveFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .setFetcherClass(FileSystemFetcher.class.getName())
- .setFetcherConfigJson(OBJECT_MAPPER.writeValueAsString(ImmutableMap
- .builder()
- .put("basePath", targetFolder)
- .put("extractFileSystemMetadata", false)
- .build()))
- .build());
- assertEquals(fetcherId, reply.getFetcherId());
- GetFetcherReply getFetcherReply = blockingStub.getFetcher(GetFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .build());
- assertEquals("false", getFetcherReply
- .getParamsMap()
- .get("extractFileSystemMetadata"));
- }
-
- // get fetchers
- for (int i = 0; i < NUM_FETCHERS_TO_CREATE; ++i) {
- String fetcherId = createFetcherId(i);
- GetFetcherReply getFetcherReply = blockingStub.getFetcher(GetFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .build());
- assertEquals(fetcherId, getFetcherReply.getFetcherId());
- assertEquals(FileSystemFetcher.class.getName(), getFetcherReply.getFetcherClass());
- }
-
- // delete fetchers
- for (int i = 0; i < NUM_FETCHERS_TO_CREATE; ++i) {
- String fetcherId = createFetcherId(i);
- DeleteFetcherReply deleteFetcherReply = blockingStub.deleteFetcher(DeleteFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .build());
- Assertions.assertTrue(deleteFetcherReply.getSuccess());
- StatusRuntimeException statusRuntimeException = Assertions.assertThrows(StatusRuntimeException.class, () -> blockingStub.getFetcher(GetFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .build()));
- Assertions.assertEquals(Status.NOT_FOUND
- .getCode()
- .value(), statusRuntimeException
- .getStatus()
- .getCode()
- .value());
- }
- }
-
- @NotNull
- private static String createFetcherId(int i) {
- return "nick" + i + ":is:cool:super/" + FileSystemFetcher.class;
- }
-
- @Test
- public void testBiStream(Resources resources) throws Exception {
- String serverName = InProcessServerBuilder.generateName();
-
- Server server = InProcessServerBuilder
- .forName(serverName)
- .directExecutor()
- .addService(new TikaGrpcServerImpl(tikaConfigXml.getAbsolutePath()))
- .build()
- .start();
- resources.register(server, Duration.ofSeconds(10));
-
- ManagedChannel channel = InProcessChannelBuilder
- .forName(serverName)
- .directExecutor()
- .build();
- resources.register(channel, Duration.ofSeconds(10));
- TikaGrpc.TikaBlockingStub blockingStub = TikaGrpc.newBlockingStub(channel);
- TikaGrpc.TikaStub tikaStub = TikaGrpc.newStub(channel);
-
- String fetcherId = createFetcherId(1);
- String targetFolder = new File("target").getAbsolutePath();
- SaveFetcherReply reply = blockingStub.saveFetcher(SaveFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .setFetcherClass(FileSystemFetcher.class.getName())
- .setFetcherConfigJson(OBJECT_MAPPER.writeValueAsString(ImmutableMap
- .builder()
- .put("basePath", targetFolder)
- .put("extractFileSystemMetadata", true)
- .build()))
- .build());
-
- assertEquals(fetcherId, reply.getFetcherId());
-
- List<FetchAndParseReply> successes = Collections.synchronizedList(new ArrayList<>());
- List<FetchAndParseReply> errors = Collections.synchronizedList(new ArrayList<>());
- AtomicBoolean finished = new AtomicBoolean(false);
-
- StreamObserver<FetchAndParseReply> replyStreamObserver = new StreamObserver<>() {
- @Override
- public void onNext(FetchAndParseReply fetchAndParseReply) {
- LOG.debug("Fetched {} with metadata {}", fetchAndParseReply.getFetchKey(), fetchAndParseReply.getFieldsMap());
- if (PipesResult.STATUS.FETCH_EXCEPTION.name().equals(fetchAndParseReply.getStatus())) {
- errors.add(fetchAndParseReply);
- } else {
- successes.add(fetchAndParseReply);
- }
- }
-
- @Override
- public void onError(Throwable throwable) {
- fail(throwable);
- }
-
- @Override
- public void onCompleted() {
- LOG.info("Stream completed");
- finished.set(true);
- }
- };
-
- StreamObserver<FetchAndParseRequest> requestStreamObserver = tikaStub.fetchAndParseBiDirectionalStreaming(replyStreamObserver);
-
- File testDocumentFolder = new File("target/" + DateTimeFormatter
- .ofPattern("yyyy_MM_dd_HH_mm_ssSSS", Locale.getDefault())
- .format(LocalDateTime.now(ZoneId.systemDefault())) + "-" + UUID.randomUUID());
- assertTrue(testDocumentFolder.mkdir());
- try {
- for (int i = 0; i < NUM_TEST_DOCS; ++i) {
- File testFile = new File(testDocumentFolder, "test-" + i + ".html");
- FileUtils.writeStringToFile(testFile, "<html><body>test " + i + "</body></html>", StandardCharsets.UTF_8);
- }
- File[] testDocuments = testDocumentFolder.listFiles();
- assertNotNull(testDocuments);
- for (File testDocument : testDocuments) {
- requestStreamObserver.onNext(FetchAndParseRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .setFetchKey(testDocument.getAbsolutePath())
- .build());
- }
- // Now test error condition
- requestStreamObserver.onNext(FetchAndParseRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .setFetchKey("does not exist")
- .build());
- requestStreamObserver.onCompleted();
- assertEquals(NUM_TEST_DOCS, successes.size());
- assertEquals(1, errors.size());
- assertTrue(finished.get());
- } finally {
- FileUtils.deleteDirectory(testDocumentFolder);
- }
- }
-}
diff --git a/tika-grpc/src/test/resources/certs/README.md b/tika-grpc/src/test/resources/certs/README.md
deleted file mode 100644
index 7373d5635..000000000
--- a/tika-grpc/src/test/resources/certs/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# Test certs for Tika Grpc mTLS
-
-Generate these using script as documented here:
-
-https://github.com/grpc/grpc-java/tree/b3ffb5078df361d7460786e134db7b5c00939246/testing/src/main/resources/certs
diff --git a/tika-grpc/src/test/resources/certs/ca.key b/tika-grpc/src/test/resources/certs/ca.key
deleted file mode 100644
index 03be0bfa6..000000000
--- a/tika-grpc/src/test/resources/certs/ca.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCwYvShd+UXQvOg
-z4GH6pRT3KGrPDbDw45fma7+I0LJQ4GupoeLuYYfHvcYPTV2I3MLO+VxCp00gfo1
-BIvsNOkGNxrrqNhP27ve9l7YwOuvWdVu4u9+73znRx3GJQ4ie/nF/z6xMcbQL5r5
-UC8yGwuJGOyr6VcpEnKTnORtuwRPJuqnGgn4rsKhLLfJz+RAhjdOKnAS3CQo/iHP
-KjoqIZ38M97GJ7icFQic3dtLUFR41nnN5ogLZ6DduR55btypPnlv5h6foLFjRMST
-MEroAq39ZSJqUoyBPTBtPFFk7uRQIfdKrp7/Bd4V0n4e91Us+UCDlOcxo2lF1CKH
-/ydEWmx3AgMBAAECggEAKrDosKQKKKUlvkg6+6CFIf8GiiFax+ru7KiPuCbkpT3X
-h2P67pCKq8Gc4Jr/84YE9DUdBU0iW3ESE/7ztsnflIeF1n/ZSwrN39sVfbTD1n8R
-r3LxsHFac8e8pxaU4zfKbmemztBTZFQBWFJV+fSdyCLmNX2WgPRcEuooR366PkWv
-xZLAxeDGqpnsa62o1GdGmalxx8aljLN/QcbQi73mR9Osim1OtSd1cyDlZ/8x6OoV
-Ae5GDN3Bj0hO9ZKzNWTbQpRw9SHKU6sWXtHlcDx4xi5kN/n9aptn7kixbY9y8uOM
-5zjErVGWvxdP94IvlSkrkenwnIjlHBtdlAjVuCFioQKBgQDoJLyfHNWPBnjPGVnK
-xcbIIwmf4C9UnZBbHRD3YxU/GBpsPgPh9EwhQTAXlGQGHeuslxCIh4cEfbOIrJ9b
-/s3OqeL9CSUaz/N+1av1ZuwOI9CEvNPi51IK+rXNRmVJG8pG6RaKNx57pXaFtmqq
-FUtC7twbPECvjspapn61nZYSiQKBgQDCg1tpGwZJJOCIkhYH4wFc4j4p0LxIcBJ2
-E3L9VnQ+APT/x8uitkZsuRY9tmWcHK8/zWTc1GpFdwGUJ9+Yzvprtej+P/buxM9J
-Y6ZJZdCIHWDuh3eq+sXS4lwr5fi7ir5m97npG1bXPlOoYIJ7p172EyoNmurRIgiP
-LWnzK0jG/wKBgQCRQtOouNFFcyZLaTCPutxdRddy7ESRrRq0eOax9pVH6tw12URy
-snyk3naqepdwYG6li82zsSKig8nA/0uktDeyVwoLjhpiwbc7KZc1sxaI7o4/US1B
-McBb0G/MqH0elz4myxnomP8BHhOhLflmvnZexrqCbFyJvk8PFFn7aUWMCQKBgDvX
-9BCzOszYJqh94X9NrQapqJxu1u6mZFelhjRBHARTgQ0MqC8IS0R58UjNTBeqj5Re
-mdCDHar/gSHW3qkBzPPEhMlsXol5TZjzqp5cT7sA5uicDwowmxpVgCwVVeBFQG0n
-fDAmtCIGz/A2uQ5YIRQuMzr6VZJAGUgLndQtlfd7AoGBAMq1imggFKd1rt49XCnO
-t97lpWOT+TlWYblHr01tOw+esawG5MFucqVI6tGpBSccTRQw6orWf4GK3KmkgQ6J
-UgHKjwYsA0sf4U5vppkdkbAbM/WwUPOTQpGFRERyJqMqFGIc4wMtZOJBxXwf+9iD
-l8tvan8w/6HugqnI7qqkTgLq
------END PRIVATE KEY-----
diff --git a/tika-grpc/src/test/resources/certs/ca.pem b/tika-grpc/src/test/resources/certs/ca.pem
deleted file mode 100644
index 49d39cd8e..000000000
--- a/tika-grpc/src/test/resources/certs/ca.pem
+++ /dev/null
@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDWjCCAkKgAwIBAgIUWrP0VvHcy+LP6UuYNtiL9gBhD5owDQYJKoZIhvcNAQEL
-BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
-GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTIw
-MDMxNzE4NTk1MVoXDTMwMDMxNTE4NTk1MVowVjELMAkGA1UEBhMCQVUxEzARBgNV
-BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
-ZDEPMA0GA1UEAwwGdGVzdGNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
-AQEAsGL0oXflF0LzoM+Bh+qUU9yhqzw2w8OOX5mu/iNCyUOBrqaHi7mGHx73GD01
-diNzCzvlcQqdNIH6NQSL7DTpBjca66jYT9u73vZe2MDrr1nVbuLvfu9850cdxiUO
-Inv5xf8+sTHG0C+a+VAvMhsLiRjsq+lXKRJyk5zkbbsETybqpxoJ+K7CoSy3yc/k
-QIY3TipwEtwkKP4hzyo6KiGd/DPexie4nBUInN3bS1BUeNZ5zeaIC2eg3bkeeW7c
-qT55b+Yen6CxY0TEkzBK6AKt/WUialKMgT0wbTxRZO7kUCH3Sq6e/wXeFdJ+HvdV
-LPlAg5TnMaNpRdQih/8nRFpsdwIDAQABoyAwHjAMBgNVHRMEBTADAQH/MA4GA1Ud
-DwEB/wQEAwICBDANBgkqhkiG9w0BAQsFAAOCAQEAkTrKZjBrJXHps/HrjNCFPb5a
-THuGPCSsepe1wkKdSp1h4HGRpLoCgcLysCJ5hZhRpHkRihhef+rFHEe60UePQO3S
-CVTtdJB4CYWpcNyXOdqefrbJW5QNljxgi6Fhvs7JJkBqdXIkWXtFk2eRgOIP2Eo9
-/OHQHlYnwZFrk6sp4wPyR+A95S0toZBcyDVz7u+hOW0pGK3wviOe9lvRgj/H3Pwt
-bewb0l+MhRig0/DVHamyVxrDRbqInU1/GTNCwcZkXKYFWSf92U+kIcTth24Q1gcw
-eZiLl5FfrWokUNytFElXob0V0a5/kbhiLc3yWmvWqHTpqCALbVyF+rKJo2f5Kw==
------END CERTIFICATE-----
diff --git a/tika-grpc/src/test/resources/certs/client.key b/tika-grpc/src/test/resources/certs/client.key
deleted file mode 100644
index 349b40033..000000000
--- a/tika-grpc/src/test/resources/certs/client.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCyqYRp+DXVp72N
-FbQH8hdhTZLycZXOlJhmMsrJmrjn2p7pI/8mTZ/0FC+SGWBGZV+ELiHrmCX5zfaI
-Lr9Iuw7Ghr3Vzoefi8r62rLupVPNi/qdqyjWk2dECHC9Z3+Ag3KzKTyerXWjKcvy
-KVmM0ZxE0RXhDW/RoQbqZsU2GKg1B2rhUU8KN0gVmKn0rJHOxzRVSYeYLYp5Yn7K
-rtPJcKyo9aVuEr7dGANzpyF6lg/nYBWc+9SGwkoLdFvKvABYJMyrbNhHUQfv0fza
-Z0P86dfTENrDxzALrzGnqcx3KTrwJjkZ/aSr1tyD0/tXvukRFiPxWBJhjHQ70GqT
-FQY19RbhAgMBAAECggEAIL8JUhL4awyvpWhQ8xPgTSlWwbEn8BE0TacJnCILuhNM
-BRdf8LlRk/8PKQwVpVF3TFbYSMI+U6b4hMVssfv3HVQc/083dHq+3XOwUCVlUstR
-SAzTE2E5EDMr1stdh0SQhV4Nilfos9s5Uk1Z6IGSztoz1GgOErIc/mGPy/aA/hbr
-fRWHvTp35+MbCJSvZuOeevX2iLs0dNzqdk6DiOWIH/BVGirVPtO6ykrkuTj1FWiN
-hyZ3MBChShlNH2poNX46ntOc7nEus0qteOgxBK8lummFEtlehCA7hd/8xuvYlP0k
-7aN684LCRDajmAGpoZO57NSDYQhAFGZeUZ93SMFucQKBgQDe7GGkzZFEiv91u1q9
-lgMy1h5dZjIZKgQaOarPC6wCQMUdqCf6cSLsAPr4T8EDoWsnY7dSnrTZ6YCIFL1T
-idg8M3BQXipICCJkFORS76pKKZ0wMn3/NgkSepsmNct91WHr6okvx4tOaoRCtdzU
-g7jt4Mr3sfLCiZtqTQyySdMUEwKBgQDNK+ZFKL0XhkWZP+PGKjWG8LWpPiK3d78/
-wYBFXzSTGlkr6FvRmYtZeNwXWRYLB4UxZ9At4hbJVEdi/2dITOz/sehVDyCAjjs3
-gycsc3UJqiZbcw5XKhI5TWBuWxkKENdbMSayogVbp2aSYoRblH764//t0ACmbfTW
-KUQRQPB/uwKBgQC5QjjjfPL8w4cJkGoYpFKELO2PMR7xSrmeEc6hwlFwjeNCgjy3
-JM6g0y++rIj7O2qRkY0IXFxvvF3UuWedxTCu1xC/uYHp2ti506LsScB7YZoAM/YB
-4iYn9Tx6xLoYGP0H0iGwU2SyBlNkHT8oXU+SYP5MWtYkVbeS3/VtNWz1gQKBgQCA
-6Nk4kN0mH7YxEKRzSOfyzeDF4oV7kuB2FYUbkTL+TirC3K58JiYY5Egc31trOKFm
-Jlz1xz0b6DkmKWTiV3r9OPHKJ8P7IeJxAZWmZzCdDuwkv0i+WW+z0zsIe3JjEavN
-3zb6O7R0HtziksWoqMeTqZeO+wa9iw6vVKQw1wWEqwKBgFHfahFs0DZ5cUTpGpBt
-F/AQG7ukgipB6N6AkB9kDbgCs1FLgd199MQrEncug5hfpq8QerbyMatmA+GXoGMb
-7vztKEH85yzp4n02FNL6H7xL4VVILvyZHdolmiORJ4qT2hZnl8pEQ2TYuF4RlHUd
-nSwXX+2o0J/nF85fm4AwWKAc
------END PRIVATE KEY-----
diff --git a/tika-grpc/src/test/resources/certs/client.pem b/tika-grpc/src/test/resources/certs/client.pem
deleted file mode 100644
index 8815875f3..000000000
--- a/tika-grpc/src/test/resources/certs/client.pem
+++ /dev/null
@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDNzCCAh8CFGyX00RCepOv/qCJ1oVdTtY92U83MA0GCSqGSIb3DQEBCwUAMFYx
-CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
-cm5ldCBXaWRnaXRzIFB0eSBMdGQxDzANBgNVBAMMBnRlc3RjYTAeFw0yMDAzMTgw
-MTA2MTBaFw0zMDAzMTYwMTA2MTBaMFoxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApT
-b21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEzAR
-BgNVBAMMCnRlc3RjbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
-AQCyqYRp+DXVp72NFbQH8hdhTZLycZXOlJhmMsrJmrjn2p7pI/8mTZ/0FC+SGWBG
-ZV+ELiHrmCX5zfaILr9Iuw7Ghr3Vzoefi8r62rLupVPNi/qdqyjWk2dECHC9Z3+A
-g3KzKTyerXWjKcvyKVmM0ZxE0RXhDW/RoQbqZsU2GKg1B2rhUU8KN0gVmKn0rJHO
-xzRVSYeYLYp5Yn7KrtPJcKyo9aVuEr7dGANzpyF6lg/nYBWc+9SGwkoLdFvKvABY
-JMyrbNhHUQfv0fzaZ0P86dfTENrDxzALrzGnqcx3KTrwJjkZ/aSr1tyD0/tXvukR
-FiPxWBJhjHQ70GqTFQY19RbhAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAFXCewK8
-cWT+zWxXyGFnouFSBzTi0BMBJRrhsiNoiQxkqityJHWFExiQZie+7CA+EabXCQUB
-+JwMSWM29j3mSw10DTfmC3rhheQqGxy304BZyUpdpvI2dt3p/mcsE7O+p4sQrSep
-gijiDssKAfxTAmUM93N6+Q8yJK5immxlbeYfijoBvmkzyB/B+qNRPsx0n7aFGnfv
-oWfkW296iPhWLiwknpC3xB6oK3vRbK4Zj1OaGb0grK7VN8EyhBix2xVF61i4dzCK
-kMIpl7CUpw1Mb2z8q3F2bHBS7iF7g1Ccn5VGcO+aJ+6PWydaeqJ6VEBF0Nwv9woe
-mL5AluNRLaqjZvE=
------END CERTIFICATE-----
diff --git a/tika-grpc/src/test/resources/certs/server1.key b/tika-grpc/src/test/resources/certs/server1.key
deleted file mode 100644
index 086462992..000000000
--- a/tika-grpc/src/test/resources/certs/server1.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDnE443EknxvxBq
-6+hvn/t09hl8hx366EBYvZmVM/NC+7igXRAjiJiA/mIaCvL3MS0Iz5hBLxSGICU+
-WproA3GCIFITIwcf/ETyWj/5xpgZ4AKrLrjQmmX8mhwUajfF3UvwMJrCOVqPp67t
-PtP+2kBXaqrXdvnvXR41FsIB8V7zIAuIZB6bHQhiGVlc1sgZYsE2EGG9WMmHtS86
-qkAOTjG2XyjmPTGAwhGDpYkYrpzp99IiDh4/Veai81hn0ssQkbry0XRD/Ig3jcHh
-23WiriPNJ0JsbgXUSLKRPZObA9VgOLy2aXoN84IMaeK3yy+cwSYG/99w93fUZJte
-MXwz4oYZAgMBAAECggEBAIVn2Ncai+4xbH0OLWckabwgyJ4IM9rDc0LIU368O1kU
-koais8qP9dujAWgfoh3sGh/YGgKn96VnsZjKHlyMgF+r4TaDJn3k2rlAOWcurGlj
-1qaVlsV4HiEzp7pxiDmHhWvp4672Bb6iBG+bsjCUOEk/n9o9KhZzIBluRhtxCmw5
-nw4Do7z00PTvN81260uPWSc04IrytvZUiAIx/5qxD72bij2xJ8t/I9GI8g4FtoVB
-8pB6S/hJX1PZhh9VlU6Yk+TOfOVnbebG4W5138LkB835eqk3Zz0qsbc2euoi8Hxi
-y1VGwQEmMQ63jXz4c6g+X55ifvUK9Jpn5E8pq+pMd7ECgYEA93lYq+Cr54K4ey5t
-sWMa+ye5RqxjzgXj2Kqr55jb54VWG7wp2iGbg8FMlkQwzTJwebzDyCSatguEZLuB
-gRGroRnsUOy9vBvhKPOch9bfKIl6qOgzMJB267fBVWx5ybnRbWN/I7RvMQf3k+9y
-biCIVnxDLEEYyx7z85/5qxsXg/MCgYEA7wmWKtCTn032Hy9P8OL49T0X6Z8FlkDC
-Rk42ygrc/MUbugq9RGUxcCxoImOG9JXUpEtUe31YDm2j+/nbvrjl6/bP2qWs0V7l
-dTJl6dABP51pCw8+l4cWgBBX08Lkeen812AAFNrjmDCjX6rHjWHLJcpS18fnRRkP
-V1d/AHWX7MMCgYEA6Gsw2guhp0Zf2GCcaNK5DlQab8OL4Hwrpttzo4kuTlwtqNKp
-Q9H4al9qfF4Cr1TFya98+EVYf8yFRM3NLNjZpe3gwYf2EerlJj7VLcahw0KKzoN1
-QBENfwgPLRk5sDkx9VhSmcfl/diLroZdpAwtv3vo4nEoxeuGFbKTGx3Qkf0CgYEA
-xyR+dcb05Ygm3w4klHQTowQ10s1H80iaUcZBgQuR1ghEtDbUPZHsoR5t1xCB02ys
-DgAwLv1bChIvxvH/L6KM8ovZ2LekBX4AviWxoBxJnfz/EVau98B0b1auRN6eSC83
-FRuGldlSOW1z/nSh8ViizSYE5H5HX1qkXEippvFRE88CgYB3Bfu3YQY60ITWIShv
-nNkdcbTT9eoP9suaRJjw92Ln+7ZpALYlQMKUZmJ/5uBmLs4RFwUTQruLOPL4yLTH
-awADWUzs3IRr1fwn9E+zM8JVyKCnUEM3w4N5UZskGO2klashAd30hWO+knRv/y0r
-uGIYs9Ek7YXlXIRVrzMwcsrt1w==
------END PRIVATE KEY-----
diff --git a/tika-grpc/src/test/resources/certs/server1.pem b/tika-grpc/src/test/resources/certs/server1.pem
deleted file mode 100644
index 88244f856..000000000
--- a/tika-grpc/src/test/resources/certs/server1.pem
+++ /dev/null
@@ -1,22 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDtDCCApygAwIBAgIUbJfTREJ6k6/+oInWhV1O1j3ZT0IwDQYJKoZIhvcNAQEL
-BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
-GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTIw
-MDMxODAzMTA0MloXDTMwMDMxNjAzMTA0MlowZTELMAkGA1UEBhMCVVMxETAPBgNV
-BAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRUwEwYDVQQKDAxFeGFtcGxl
-LCBDby4xGjAYBgNVBAMMESoudGVzdC5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0B
-AQEFAAOCAQ8AMIIBCgKCAQEA5xOONxJJ8b8Qauvob5/7dPYZfIcd+uhAWL2ZlTPz
-Qvu4oF0QI4iYgP5iGgry9zEtCM+YQS8UhiAlPlqa6ANxgiBSEyMHH/xE8lo/+caY
-GeACqy640Jpl/JocFGo3xd1L8DCawjlaj6eu7T7T/tpAV2qq13b5710eNRbCAfFe
-8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c
-6ffSIg4eP1XmovNYZ9LLEJG68tF0Q/yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPV
-YDi8tml6DfOCDGnit8svnMEmBv/fcPd31GSbXjF8M+KGGQIDAQABo2swaTAJBgNV
-HRMEAjAAMAsGA1UdDwQEAwIF4DBPBgNVHREESDBGghAqLnRlc3QuZ29vZ2xlLmZy
-ghh3YXRlcnpvb2kudGVzdC5nb29nbGUuYmWCEioudGVzdC55b3V0dWJlLmNvbYcE
-wKgBAzANBgkqhkiG9w0BAQsFAAOCAQEAS8hDQA8PSgipgAml7Q3/djwQ644ghWQv
-C2Kb+r30RCY1EyKNhnQnIIh/OUbBZvh0M0iYsy6xqXgfDhCB93AA6j0i5cS8fkhH
-Jl4RK0tSkGQ3YNY4NzXwQP/vmUgfkw8VBAZ4Y4GKxppdATjffIW+srbAmdDruIRM
-wPeikgOoRrXf0LA1fi4TqxARzeRwenQpayNfGHTvVF9aJkl8HoaMunTAdG5pIVcr
-9GKi/gEMpXUJbbVv3U5frX1Wo4CFo+rZWJ/LyCMeb0jciNLxSdMwj/E/ZuExlyeZ
-gc9ctPjSMvgSyXEKv6Vwobleeg88V2ZgzenziORoWj4KszG/lbQZvg==
------END CERTIFICATE-----
diff --git a/tika-grpc/src/test/resources/tika-pipes-test-config.xml b/tika-grpc/src/test/resources/tika-pipes-test-config.xml
deleted file mode 100644
index e4006edb3..000000000
--- a/tika-grpc/src/test/resources/tika-pipes-test-config.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?><!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<properties>
- <async>
- <staleFetcherTimeoutSeconds>600</staleFetcherTimeoutSeconds>
- <staleFetcherDelaySeconds>60</staleFetcherDelaySeconds>
- </async>
- <pipes>
- <params>
- <numClients>2</numClients>
- <forkedJvmArgs>
- <arg>-Xmx1g</arg>
- <arg>-XX:ParallelGCThreads=2</arg>
- </forkedJvmArgs>
- <timeoutMillis>60000</timeoutMillis>
- <maxForEmitBatchBytes>-1</maxForEmitBatchBytes> <!-- disable emit -->
- </params>
- </pipes>
- <fetchers>
- </fetchers>
-</properties>
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index 77baf60cb..e3a0e9d7a 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -439,6 +439,7 @@
<snappy.version>1.1.10.8</snappy.version>
<solrj.version>9.9.0</solrj.version>
<spring.version>6.2.11</spring.version>
+ <spring.boot.version>3.4.1</spring.boot.version>
<sqlite.version>3.50.3.0</sqlite.version>
<stax.ex.version>2.1.0</stax.ex.version>
<testcontainers.version>1.21.3</testcontainers.version>
@@ -456,6 +457,8 @@
<zstd.version>1.5.7-4</zstd.version>
<nimbus-jose-jwt.version>10.5</nimbus-jose-jwt.version>
<javacpp.version>1.5.12</javacpp.version>
+ <pf4j.version>3.12.0</pf4j.version>
+ <lombok.version>1.18.36</lombok.version>
<!-- needs to exist even if empty due to problems with jacoco-maven-plugin -->
<addmod />
@@ -479,6 +482,13 @@
<version>2.9.0</version>
</dependency>
<!-- boms -->
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-dependencies</artifactId>
+ <version>${spring.boot.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
<dependency>
<groupId>com.fasterxml.jackson</groupId>
<artifactId>jackson-bom</artifactId>
@@ -1149,6 +1159,16 @@
<artifactId>nimbus-jose-jwt</artifactId>
<version>${nimbus-jose-jwt.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.pf4j</groupId>
+ <artifactId>pf4j</artifactId>
+ <version>${pf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>${lombok.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/tika-pipes/pom.xml b/tika-pipes/pom.xml
index 204311932..cc06d9615 100644
--- a/tika-pipes/pom.xml
+++ b/tika-pipes/pom.xml
@@ -1,19 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
-->
+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -25,69 +30,17 @@
</parent>
<artifactId>tika-pipes</artifactId>
- <name>Apache Tika pipes</name>
- <url>https://tika.apache.org/</url>
<packaging>pom</packaging>
+ <name>Apache Tika Pipes</name>
+ <description>Apache Tika Pipes aggregator project</description>
<modules>
<module>tika-pipes-core</module>
- <module>tika-httpclient-commons</module>
- <module>tika-fetchers</module>
- <module>tika-emitters</module>
- <module>tika-pipes-iterators</module>
- <module>tika-pipes-reporters</module>
- <module>tika-async-cli</module>
+ <module>tika-pipes-pipe-iterators</module>
+ <module>tika-pipes-fetchers</module>
+ <module>tika-pipes-emitters</module>
+ <module>tika-pipes-proto</module>
+ <module>tika-pipes-grpc</module>
+ <module>tika-pipes-cli</module>
</modules>
- <dependencies>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <version>${checkstyle.plugin.version}</version>
- <dependencies>
- <dependency>
- <groupId>com.puppycrawl.tools</groupId>
- <artifactId>checkstyle</artifactId>
- <version>${puppycrawl.version}</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>validate</id>
- <phase>validate</phase>
- <configuration>
- <configLocation>checkstyle.xml</configLocation>
- <inputEncoding>UTF-8</inputEncoding>
- <consoleOutput>true</consoleOutput>
- <logViolationsToConsole>true</logViolationsToConsole>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <testSourceDirectories>${project.basedir}/src/test/java</testSourceDirectories>
- <violationSeverity>error</violationSeverity>
- <failOnViolation>true</failOnViolation>
- <excludeGeneratedSources>true</excludeGeneratedSources>
- </configuration>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
+</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-async-cli/pom.xml b/tika-pipes/tika-async-cli/pom.xml
deleted file mode 100644
index 47991bcc3..000000000
--- a/tika-pipes/tika-async-cli/pom.xml
+++ /dev/null
@@ -1,134 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-pipes</artifactId>
- <version>4.0.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-async-cli</artifactId>
-
- <name>Apache Tika Async CLI</name>
- <url>https://tika.apache.org/</url>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
- <!-- logging -->
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-serialization</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.reporters.fs.status</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.tika.async.cli.TikaAsyncCLI</mainClass>
- <manifestEntries>
- <Multi-Release>true</Multi-Release>
- </manifestEntries>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
diff --git a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/SimpleAsyncConfig.java b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/SimpleAsyncConfig.java
deleted file mode 100644
index 34458beb0..000000000
--- a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/SimpleAsyncConfig.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.async.cli;
-
-class SimpleAsyncConfig {
-
- private String inputDir;
- private String outputDir;
- private Integer numClients;
- private Long timeoutMs;
- private String xmx;
- private String fileList;
-
- public SimpleAsyncConfig(String inputDir, String outputDir, Integer numClients, Long timeoutMs, String xmx, String fileList) {
- this.inputDir = inputDir;
- this.outputDir = outputDir;
- this.numClients = numClients;
- this.timeoutMs = timeoutMs;
- this.xmx = xmx;
- this.fileList = fileList;
- }
-
- public String getInputDir() {
- return inputDir;
- }
-
- public String getOutputDir() {
- return outputDir;
- }
-
- public Integer getNumClients() {
- return numClients;
- }
-
- public Long getTimeoutMs() {
- return timeoutMs;
- }
-
- public String getXmx() {
- return xmx;
- }
-
- public String getFileList() {
- return fileList;
- }
-}
diff --git a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java
deleted file mode 100644
index 769265ed1..000000000
--- a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.async.cli;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.concurrent.TimeoutException;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.async.AsyncProcessor;
-import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
-
-public class TikaAsyncCLI {
-
- private static final long TIMEOUT_MS = 600_000;
- private static final Logger LOG = LoggerFactory.getLogger(TikaAsyncCLI.class);
-
- private static Options getOptions() {
- Options options = new Options();
- options.addOption("i", "inputDir", true, "input directory");
- options.addOption("o", "outputDir", true, "output directory");
-
- options.addOption("n", "numClients", true, "number of forked clients");
- options.addOption("x", "Xmx", true, "heap for the forked clients in usual jvm heap amount, e.g. -x 1g");
- options.addOption("?", "help", false, "this help message");
- options.addOption("t", "timeoutMs", true, "timeout for each parse in milliseconds");
- options.addOption("l", "fileList", true, "file list");
-
- return options;
- }
-
- public static void main(String[] args) throws Exception {
- if (args.length == 0) {
- usage(getOptions());
- } else if (args.length == 1) {
- processWithTikaConfig(Paths.get(args[0]));
- } else {
- processCommandLine(args);
- }
- }
-
- private static void processCommandLine(String[] args) throws Exception {
-
- SimpleAsyncConfig simpleAsyncConfig = parseCommandLine(args);
-
- Path tikaConfig = null;
- try {
- tikaConfig = Files.createTempFile("tika-async-tmp-", ".xml");
- TikaConfigAsyncWriter tikaConfigAsyncWriter = new TikaConfigAsyncWriter(simpleAsyncConfig);
- tikaConfigAsyncWriter.write(tikaConfig);
- processWithTikaConfig(tikaConfig);
- } finally {
- if (tikaConfig != null) {
- Files.delete(tikaConfig);
- }
- }
- }
-
- //not private for testing purposes
- static SimpleAsyncConfig parseCommandLine(String[] args) throws ParseException, IOException {
- if (args.length == 2 && ! args[0].startsWith("-")) {
- return new SimpleAsyncConfig(args[0], args[1], null, null, null, null);
- }
-
- Options options = getOptions();
-
- CommandLineParser cliParser = new DefaultParser();
-
- CommandLine line = cliParser.parse(options, args);
- if (line.hasOption("help")) {
- usage(options);
- }
- String inputDir = null;
- String outputDir = null;
- String xmx = null;
- Long timeoutMs = null;
- Integer numClients = null;
- String fileList = null;
- if (line.hasOption("i")) {
- inputDir = line.getOptionValue("i");
- }
- if (line.hasOption("o")) {
- outputDir = line.getOptionValue("o");
- }
- if (line.hasOption("x")) {
- xmx = line.getOptionValue("x");
- }
- if (line.hasOption("t")) {
- timeoutMs = Long.parseLong(line.getOptionValue("t"));
- }
- if (line.hasOption("n")) {
- numClients = Integer.parseInt(line.getOptionValue("n"));
- }
- if (line.hasOption("l")) {
- fileList = line.getOptionValue("l");
- }
- return new SimpleAsyncConfig(inputDir, outputDir,
- numClients, timeoutMs, xmx, fileList);
- }
-
-
- private static void processWithTikaConfig(Path tikaConfigPath) throws Exception {
- PipesIterator pipesIterator = PipesIterator.build(tikaConfigPath);
- long start = System.currentTimeMillis();
- try (AsyncProcessor processor = new AsyncProcessor(tikaConfigPath, pipesIterator)) {
-
- for (FetchEmitTuple t : pipesIterator) {
- boolean offered = processor.offer(t, TIMEOUT_MS);
- if (!offered) {
- throw new TimeoutException("timed out waiting to add a fetch emit tuple");
- }
- }
- processor.finished();
- while (true) {
- if (processor.checkActive()) {
- Thread.sleep(500);
- } else {
- break;
- }
- }
- long elapsed = System.currentTimeMillis() - start;
- LOG.info("Successfully finished processing {} files in {} ms", processor.getTotalProcessed(), elapsed);
-
- }
- }
-
- private static void usage(Options options) throws IOException {
- System.out.println("Two primary options:");
- System.out.println("\t1. Specify a tika-config.xml on the commandline that includes the definitions for async");
- System.out.println("\t2. Commandline:");
- org.apache.commons.cli.help.HelpFormatter helpFormatter = org.apache.commons.cli.help.HelpFormatter.builder().get();
- helpFormatter.printHelp("tikaAsynCli", null, options, null, true);
- System.exit(1);
- }
-}
diff --git a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaConfigAsyncWriter.java b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaConfigAsyncWriter.java
deleted file mode 100644
index d24141023..000000000
--- a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaConfigAsyncWriter.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.async.cli;
-
-import java.io.IOException;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-
-import org.apache.tika.utils.StringUtils;
-
-class TikaConfigAsyncWriter {
-
- private static final String FETCHER_NAME = "fsf";
- private static final String EMITTER_NAME = "fse";
-
- private final SimpleAsyncConfig simpleAsyncConfig;
-
- TikaConfigAsyncWriter(SimpleAsyncConfig simpleAsyncConfig) {
- this.simpleAsyncConfig = simpleAsyncConfig;
- }
-
- void write(Path output) throws IOException {
- try {
- _write(output);
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- void _write(Path output) throws ParserConfigurationException, TransformerException, IOException {
- DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- Document document = dbf.newDocumentBuilder().newDocument();
- Element properties = document.createElement("properties");
- document.appendChild(properties);
- writePipesIterator(document, properties);
- writeFetchers(document, properties);
- writeEmitters(document, properties);
- writeAsync(document, properties);
- Transformer transformer = TransformerFactory
- .newInstance().newTransformer();
- transformer.setOutputProperty(OutputKeys.INDENT, "yes");
- transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
- try (Writer writer = Files.newBufferedWriter(output, StandardCharsets.UTF_8)) {
- StreamResult result = new StreamResult(writer);
- DOMSource source = new DOMSource(document);
- transformer.transform(source, result);
- }
-
- }
-
- private void writePipesIterator(Document document, Element properties) {
- if (! StringUtils.isBlank(simpleAsyncConfig.getFileList())) {
- writeFileListIterator(document, properties);
- } else {
- writeFileSystemIterator(document, properties);
- }
- }
-
- private void writeFileSystemIterator(Document document, Element properties) {
- Element pipesIterator = createAndGetElement(document, properties, "pipesIterator",
- "class", "org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator");
- appendTextElement(document, pipesIterator, "basePath",
- Paths.get(simpleAsyncConfig.getInputDir()).toAbsolutePath().toString());
- appendTextElement(document, pipesIterator, "fetcherName", FETCHER_NAME);
- appendTextElement(document, pipesIterator, "emitterName", EMITTER_NAME);
- }
-
- private void writeFileListIterator(Document document, Element properties) {
- Element pipesIterator = createAndGetElement(document, properties, "pipesIterator",
- "class", "org.apache.tika.pipes.pipesiterator.filelist.FileListPipesIterator");
- appendTextElement(document, pipesIterator, "fetcherName", FETCHER_NAME);
- appendTextElement(document, pipesIterator, "emitterName", EMITTER_NAME);
- appendTextElement(document, pipesIterator, "fileList",
- Paths.get(simpleAsyncConfig.getFileList()).toAbsolutePath().toString());
- appendTextElement(document, pipesIterator, "hasHeader", "false");
- }
-
- private void writeEmitters(Document document, Element properties) {
- Element emitters = createAndGetElement(document, properties, "emitters");
- Element emitter = createAndGetElement( document, emitters, "emitter",
- "class", "org.apache.tika.pipes.core.emitter.fs.FileSystemEmitter");
- appendTextElement(document, emitter, "name", EMITTER_NAME);
- appendTextElement(document, emitter, "basePath",
- Paths.get(simpleAsyncConfig.getOutputDir()).toAbsolutePath().toString());
- }
-
- private void writeFetchers(Document document, Element properties) {
- Element fetchers = createAndGetElement(document, properties, "fetchers");
- Element fetcher = createAndGetElement(document, fetchers, "fetcher",
- "class", "org.apache.tika.pipes.fetcher.fs.FileSystemFetcher");
- appendTextElement(document, fetcher, "name", FETCHER_NAME);
- if (!StringUtils.isBlank(simpleAsyncConfig.getInputDir())) {
- appendTextElement(document, fetcher, "basePath", Paths
- .get(simpleAsyncConfig.getInputDir())
- .toAbsolutePath()
- .toString());
- } else {
- appendTextElement(document, fetcher, "basePath", "");
- }
- }
-
- private void writeAsync(Document document, Element properties) {
- Element async = createAndGetElement(document, properties, "async");
- properties.appendChild(async);
- if (simpleAsyncConfig.getNumClients() != null) {
- appendTextElement(document, async, "numClients", Integer.toString(simpleAsyncConfig.getNumClients()));
- }
- if (simpleAsyncConfig.getXmx() != null) {
- Element forkedJvmArgs = createAndGetElement(document, async, "forkedJvmArgs");
- appendTextElement(document, forkedJvmArgs, "arg", "-Xmx" + simpleAsyncConfig.getXmx());
- }
- if (simpleAsyncConfig.getTimeoutMs() != null) {
- appendTextElement(document, async, "timeoutMillis", Long.toString(simpleAsyncConfig.getTimeoutMs()));
- }
- }
-
- private static void appendTextElement(Document document, Element parent, String itemName, String text, String... attrs) {
- Element el = createAndGetElement(document, parent, itemName, attrs);
- el.setTextContent(text);
- }
-
- private static Element createAndGetElement(Document document, Element parent, String elementName, String... attrs) {
- Element el = document.createElement(elementName);
- parent.appendChild(el);
- for (int i = 0; i < attrs.length; i += 2) {
- el.setAttribute(attrs[i], attrs[i + 1]);
- }
- return el;
- }
-
-}
diff --git a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java
deleted file mode 100644
index 4e38aac9c..000000000
--- a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.async.cli;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNull;
-
-import org.junit.jupiter.api.Test;
-
-public class AsyncCliParserTest {
-
- @Test
- public void testBasic() throws Exception {
- SimpleAsyncConfig simpleAsyncConfig = TikaAsyncCLI.parseCommandLine(new String[]{"input", "output"});
- assertEquals("input", simpleAsyncConfig.getInputDir());
- assertEquals("output", simpleAsyncConfig.getOutputDir());
- assertNull(simpleAsyncConfig.getFileList());
- assertNull(simpleAsyncConfig.getNumClients());
- assertNull(simpleAsyncConfig.getTimeoutMs());
- assertNull(simpleAsyncConfig.getXmx());
-
- simpleAsyncConfig = TikaAsyncCLI.parseCommandLine(new String[]{"-o", "output", "-i", "input"});
- assertEquals("input", simpleAsyncConfig.getInputDir());
- assertEquals("output", simpleAsyncConfig.getOutputDir());
- assertNull(simpleAsyncConfig.getFileList());
- assertNull(simpleAsyncConfig.getNumClients());
- assertNull(simpleAsyncConfig.getTimeoutMs());
- assertNull(simpleAsyncConfig.getXmx());
-
- simpleAsyncConfig = TikaAsyncCLI.parseCommandLine(new String[]{"-output", "output", "-input", "input"});
- assertEquals("input", simpleAsyncConfig.getInputDir());
- assertEquals("output", simpleAsyncConfig.getOutputDir());
- assertNull(simpleAsyncConfig.getFileList());
- assertNull(simpleAsyncConfig.getNumClients());
- assertNull(simpleAsyncConfig.getTimeoutMs());
- assertNull(simpleAsyncConfig.getXmx());
-
- simpleAsyncConfig = TikaAsyncCLI.parseCommandLine(new String[]{"--output", "output", "--input", "input"});
- assertEquals("input", simpleAsyncConfig.getInputDir());
- assertEquals("output", simpleAsyncConfig.getOutputDir());
- assertNull(simpleAsyncConfig.getFileList());
- assertNull(simpleAsyncConfig.getNumClients());
- assertNull(simpleAsyncConfig.getTimeoutMs());
- assertNull(simpleAsyncConfig.getXmx());
-
- simpleAsyncConfig = TikaAsyncCLI.parseCommandLine(new String[]{"--output=output", "--input=input"});
- assertEquals("input", simpleAsyncConfig.getInputDir());
- assertEquals("output", simpleAsyncConfig.getOutputDir());
- assertNull(simpleAsyncConfig.getFileList());
- assertNull(simpleAsyncConfig.getNumClients());
- assertNull(simpleAsyncConfig.getTimeoutMs());
- assertNull(simpleAsyncConfig.getXmx());
- }
-
- @Test
- public void testAll() throws Exception {
- SimpleAsyncConfig simpleAsyncConfig = TikaAsyncCLI.parseCommandLine(
- new String[]{"-i", "input", "-o", "output", "-n", "5", "-t", "30000", "-x", "1g"});
- assertEquals("input", simpleAsyncConfig.getInputDir());
- assertEquals("output", simpleAsyncConfig.getOutputDir());
- assertNull(simpleAsyncConfig.getFileList());
- assertEquals(5, simpleAsyncConfig.getNumClients());
- assertEquals(30000L, simpleAsyncConfig.getTimeoutMs());
- assertEquals("1g", simpleAsyncConfig.getXmx());
- }
-
- //TODO -- test for file list with and without inputDir
-}
diff --git a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncProcessorTest.java b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncProcessorTest.java
deleted file mode 100644
index 482e20ea2..000000000
--- a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncProcessorTest.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.async.cli;
-
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.List;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-import org.apache.tika.TikaTest;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.HandlerConfig;
-import org.apache.tika.pipes.core.async.AsyncProcessor;
-import org.apache.tika.pipes.core.emitter.EmitKey;
-import org.apache.tika.pipes.core.extractor.EmbeddedDocumentBytesConfig;
-import org.apache.tika.pipes.core.fetcher.FetchKey;
-import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
-import org.apache.tika.serialization.JsonMetadataList;
-
-/**
- * This should be in tika-core, but we want to avoid a dependency mess with tika-serialization
- */
-public class AsyncProcessorTest extends TikaTest {
- //TODO -- integrate json pipes iterator and run with AyncProcessor.main
- @TempDir
- private Path basedir;
- private Path inputDir;
-
- private Path bytesDir;
-
- private Path jsonDir;
-
- private Path configDir;
-
- @BeforeEach
- public void setUp() throws IOException {
- inputDir = basedir.resolve("input");
-
- bytesDir = basedir.resolve("bytes");
-
- jsonDir = basedir.resolve("json");
-
- configDir = basedir.resolve("config");
- Path tikaConfig = configDir.resolve("tika-config.xml");
-
- Files.createDirectories(basedir);
- Files.createDirectories(configDir);
- Files.createDirectories(inputDir);
-
- String xml = IOUtils.toString(AsyncProcessorTest.class.getResourceAsStream("/configs/TIKA-4207-emitter.xml"), StandardCharsets.UTF_8);
- //do stuff to xml
- xml = xml.replace("BASE_PATH", inputDir
- .toAbsolutePath()
- .toString());
- xml = xml.replace("JSON_PATH", jsonDir
- .toAbsolutePath()
- .toString());
- xml = xml.replace("BYTES_PATH", bytesDir
- .toAbsolutePath()
- .toString());
-
- Files.writeString(tikaConfig, xml, StandardCharsets.UTF_8);
-
- Path mock = inputDir.resolve("mock.xml");
- try (OutputStream os = Files.newOutputStream(mock)) {
- IOUtils.copy(getClass().getResourceAsStream("/test-documents/basic_embedded.xml"), os);
- }
- }
-
- @Test
- public void testBasic() throws Exception {
-// TikaAsyncCLI cli = new TikaAsyncCLI();
- // cli.main(new String[]{ configDir.resolve("tika-config.xml").toAbsolutePath().toString()});
- AsyncProcessor processor = new AsyncProcessor(configDir.resolve("tika-config.xml"));
-
- EmbeddedDocumentBytesConfig embeddedDocumentBytesConfig = new EmbeddedDocumentBytesConfig(true);
- embeddedDocumentBytesConfig.setIncludeOriginal(true);
- embeddedDocumentBytesConfig.setEmitter("bytes");
- embeddedDocumentBytesConfig.setSuffixStrategy(EmbeddedDocumentBytesConfig.SUFFIX_STRATEGY.NONE);
- embeddedDocumentBytesConfig.setEmbeddedIdPrefix("-");
- ParseContext parseContext = new ParseContext();
- parseContext.set(HandlerConfig.class, HandlerConfig.DEFAULT_HANDLER_CONFIG);
- parseContext.set(EmbeddedDocumentBytesConfig.class, embeddedDocumentBytesConfig);
- FetchEmitTuple t =
- new FetchEmitTuple("myId-1", new FetchKey("fs", "mock.xml"), new EmitKey("json", "emit-1"), new Metadata(), parseContext, FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT);
-
- processor.offer(t, 1000);
-
- for (int i = 0; i < 10; i++) {
- processor.offer(PipesIterator.COMPLETED_SEMAPHORE, 1000);
- }
- //TODO clean this up
- while (processor.checkActive()) {
- Thread.sleep(100);
- }
- processor.close();
-
- String container = Files.readString(bytesDir.resolve("emit-1/emit-1-0"));
- assertContains("\"dc:creator\">Nikolai Lobachevsky", container);
-
- String xmlEmbedded = Files.readString(bytesDir.resolve("emit-1/emit-1-1"));
- assertContains("name=\"dc:creator\"", xmlEmbedded);
- assertContains(">embeddedAuthor</metadata>", xmlEmbedded);
-
- List<Metadata> metadataList;
- try (BufferedReader reader = Files.newBufferedReader(jsonDir.resolve("emit-1.json"))) {
- metadataList = JsonMetadataList.fromJson(reader);
- }
- assertEquals(2, metadataList.size());
- assertContains("main_content", metadataList
- .get(0)
- .get(TikaCoreProperties.TIKA_CONTENT));
- assertContains("some_embedded_content", metadataList
- .get(1)
- .get(TikaCoreProperties.TIKA_CONTENT));
- }
-}
diff --git a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaAsyncCLITest.java b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaAsyncCLITest.java
deleted file mode 100644
index 2fd9818f2..000000000
--- a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaAsyncCLITest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.async.cli;
-
-import static org.junit.jupiter.api.Assertions.assertThrows;
-
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.exception.TikaConfigException;
-
-public class TikaAsyncCLITest {
- @Test
- public void testCrash() throws Exception {
- Path config = getPath("/configs/tika-config-broken.xml");
- assertThrows(TikaConfigException.class, () -> TikaAsyncCLI.main(new String[]{config.toAbsolutePath().toString()}));
- }
-
- private Path getPath(String file) throws Exception {
- return Paths.get(this
- .getClass()
- .getResource(file)
- .toURI());
- }
-}
diff --git a/tika-pipes/tika-async-cli/src/test/resources/configs/TIKA-4207-emitter.xml b/tika-pipes/tika-async-cli/src/test/resources/configs/TIKA-4207-emitter.xml
deleted file mode 100644
index bb8bfe77f..000000000
--- a/tika-pipes/tika-async-cli/src/test/resources/configs/TIKA-4207-emitter.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<properties>
- <fetchers>
- <fetcher class="org.apache.tika.pipes.core.fetcher.fs.FileSystemFetcher">
- <name>fs</name>
- <basePath>BASE_PATH</basePath>
- </fetcher>
- </fetchers>
- <emitters>
- <emitter class="org.apache.tika.pipes.core.emitter.fs.FileSystemEmitter">
- <name>json</name>
- <basePath>JSON_PATH</basePath>
- </emitter>
- <emitter class="org.apache.tika.pipes.core.emitter.fs.FileSystemEmitter">
- <name>bytes</name>
- <basePath>BYTES_PATH</basePath>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-async-cli/src/test/resources/configs/tika-config-broken.xml b/tika-pipes/tika-async-cli/src/test/resources/configs/tika-config-broken.xml
deleted file mode 100644
index 5ee379e6f..000000000
--- a/tika-pipes/tika-async-cli/src/test/resources/configs/tika-config-broken.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <fetchers>
- <fetcher class="org.apache.tika.pipes.fetcher.s3.S3Fetcher">
- <name>s3</name>
- <region>us-east-1</region>
- <profile><!-- fill in here --></profile>
- </fetcher>
- </fetchers>
- <pipesIterator class="org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator">
- <fetcherName>fs</fetcherName>
- <basePath>basePath</basePath>
- </pipesIterator>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-async-cli/src/test/resources/test-documents/basic_embedded.xml b/tika-pipes/tika-async-cli/src/test/resources/test-documents/basic_embedded.xml
deleted file mode 100644
index 7536a1603..000000000
--- a/tika-pipes/tika-async-cli/src/test/resources/test-documents/basic_embedded.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<mock>
-
- <metadata action="add" name="dc:creator">Nikolai Lobachevsky</metadata>
- <write element="p">main_content</write>
- <!-- auto detection wasn't working for some reason; add content-type as
- is to trigger mock on the embedded -->
- <embedded filename="embed1.xml" content-type="application/mock+xml">
- <mock>
- <metadata action="add" name="dc:creator">embeddedAuthor</metadata>
- <write element="p">some_embedded_content</write>
- </mock>
- </embedded>
-
-</mock>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/pom.xml b/tika-pipes/tika-emitters/pom.xml
deleted file mode 100644
index b63c797e1..000000000
--- a/tika-pipes/tika-emitters/pom.xml
+++ /dev/null
@@ -1,47 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-pipes</artifactId>
- <version>4.0.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitters</artifactId>
- <name>Apache Tika emitters</name>
- <url>https://tika.apache.org/</url>
- <packaging>pom</packaging>
-
- <modules>
- <module>tika-emitter-s3</module>
- <module>tika-emitter-kafka</module>
- <module>tika-emitter-solr</module>
- <module>tika-emitter-opensearch</module>
- <module>tika-emitter-gcs</module>
- <module>tika-emitter-az-blob</module>
- <module>tika-emitter-jdbc</module>
- </modules>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-az-blob/pom.xml b/tika-pipes/tika-emitters/tika-emitter-az-blob/pom.xml
deleted file mode 100644
index 5b3e1eb05..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-az-blob/pom.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-az-blob</artifactId>
- <name>Apache Tika Azure blob</name>
-
-
- <dependencies>
- <!-- should serialization be provided or bundled? -->
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-serialization</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.azure</groupId>
- <artifactId>azure-storage-blob</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.emitter.azblob</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/main/java/org/apache/tika/pipes/emitter/azblob/AZBlobEmitter.java b/tika-pipes/tika-emitters/tika-emitter-az-blob/src/main/java/org/apache/tika/pipes/emitter/azblob/AZBlobEmitter.java
deleted file mode 100644
index a4e81f2cc..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/main/java/org/apache/tika/pipes/emitter/azblob/AZBlobEmitter.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.azblob;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Path;
-import java.util.List;
-import java.util.Map;
-
-import com.azure.core.credential.AzureSasCredential;
-import com.azure.storage.blob.BlobClient;
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.BlobServiceClient;
-import com.azure.storage.blob.BlobServiceClientBuilder;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream;
-import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.StreamEmitter;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-import org.apache.tika.serialization.JsonMetadataList;
-import org.apache.tika.utils.StringUtils;
-
-
-/**
- * Emit files to Azure blob storage. Must set endpoint, sasToken and container via config.
- */
-
-public class AZBlobEmitter extends AbstractEmitter implements Initializable, StreamEmitter {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(AZBlobEmitter.class);
- private String fileExtension = "json";
-
- private String prefix = "";
-
- private String sasToken;
- private String container;
- private String endpoint;
- private BlobServiceClient blobServiceClient;
- private BlobContainerClient blobContainerClient;
- private boolean overwriteExisting = false;
-
- /**
- * Requires the src-bucket/path/to/my/file.txt in the {@link TikaCoreProperties#SOURCE_PATH}.
- *
- * @param emitKey
- * @param metadataList
- * @param parseContext
- * @throws IOException
- * @throws TikaEmitterException
- */
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext) throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.isEmpty()) {
- throw new TikaEmitterException("metadata list must not be null or of size 0");
- }
- //TODO: estimate size of metadata list. Above a certain size,
- //create a temp file?
- UnsynchronizedByteArrayOutputStream bos = UnsynchronizedByteArrayOutputStream
- .builder()
- .get();
- try (Writer writer = new OutputStreamWriter(bos, StandardCharsets.UTF_8)) {
- JsonMetadataList.toJson(metadataList, writer);
- } catch (IOException e) {
- throw new TikaEmitterException("can't jsonify", e);
- }
- Metadata metadata = new Metadata();
- emit(emitKey, TikaInputStream.get(bos.toByteArray(), metadata), metadata, parseContext);
-
- }
-
- /**
- * @param path object path; prefix will be prepended
- * @param is inputStream to copy, if a TikaInputStream contains an underlying file,
- * the client will upload the file; if a content-length is included in the
- * metadata, the client will upload the stream with the content-length;
- * otherwise, the client will copy the stream to a byte array and then
- * upload.
- * @param userMetadata this will be written to the az blob's properties.metadata
- * @param parseContext
- * @throws IOException if there is a Runtime client exception
- * @throws TikaEmitterException if there is a Runtime client exception
- */
- @Override
- public void emit(String path, InputStream is, Metadata userMetadata, ParseContext parseContext) throws IOException, TikaEmitterException {
- String lengthString = userMetadata.get(Metadata.CONTENT_LENGTH);
- long length = -1;
- if (lengthString != null) {
- try {
- length = Long.parseLong(lengthString);
- } catch (NumberFormatException e) {
- LOGGER.warn("Bad content-length: " + lengthString);
- }
- }
- if (is instanceof TikaInputStream && ((TikaInputStream) is).hasFile()) {
- write(path, userMetadata, ((TikaInputStream) is).getPath());
- } else if (length > -1) {
- LOGGER.debug("relying on the content-length set in the metadata object: {}", length);
- write(path, userMetadata, is, length);
- } else {
- try (UnsynchronizedByteArrayOutputStream bos = UnsynchronizedByteArrayOutputStream
- .builder()
- .get()) {
- IOUtils.copy(is, bos);
- write(path, userMetadata, bos.toByteArray());
- }
- }
- }
-
- private void write(String path, Metadata userMetadata, InputStream is, long length) {
- String actualPath = getActualPath(path);
- LOGGER.debug("about to emit to target container: ({}) path:({})", container, actualPath);
- BlobClient blobClient = blobContainerClient.getBlobClient(actualPath);
- updateMetadata(blobClient, userMetadata);
- blobClient.upload(is, length, overwriteExisting);
- }
-
- private void write(String path, Metadata userMetadata, Path file) {
- String actualPath = getActualPath(path);
- LOGGER.debug("about to emit to target container: ({}) path:({})", container, actualPath);
- BlobClient blobClient = blobContainerClient.getBlobClient(actualPath);
- updateMetadata(blobClient, userMetadata);
-
- blobClient.uploadFromFile(file
- .toAbsolutePath()
- .toString(), overwriteExisting);
- }
-
- private void write(String path, Metadata userMetadata, byte[] bytes) throws IOException {
- String actualPath = getActualPath(path);
- LOGGER.debug("about to emit to target container: ({}) path:({})", container, actualPath);
- BlobClient blobClient = blobContainerClient.getBlobClient(actualPath);
- updateMetadata(blobClient, userMetadata);
- blobClient.upload(UnsynchronizedByteArrayInputStream.builder().setByteArray(bytes).get(), bytes.length, overwriteExisting);
- }
-
- private void updateMetadata(BlobClient blobClient, Metadata userMetadata) {
- for (String n : userMetadata.names()) {
- if (n.equals(Metadata.CONTENT_LENGTH)) {
- continue;
- }
- String[] vals = userMetadata.getValues(n);
- if (vals.length > 1) {
- LOGGER.warn("Can only write the first value for key {}. I see {} values.", n, vals.length);
- }
- blobClient
- .getProperties()
- .getMetadata()
- .put(n, vals[0]);
- }
-
- }
-
- private String getActualPath(final String path) {
- String ret;
- if (!StringUtils.isBlank(prefix)) {
- ret = prefix + "/" + path;
- } else {
- ret = path;
- }
-
- if (!StringUtils.isBlank(fileExtension)) {
- ret += "." + fileExtension;
- }
- return ret;
- }
-
- @Field
- public void setSasToken(String sasToken) {
- this.sasToken = sasToken;
- }
-
- @Field
- public void setEndpoint(String endpoint) {
- this.endpoint = endpoint;
- }
-
- @Field
- public void setContainer(String container) {
- this.container = container;
- }
-
- @Field
- public void setOverwriteExisting(boolean overwriteExisting) {
- this.overwriteExisting = overwriteExisting;
- }
-
- @Field
- public void setPrefix(String prefix) {
- //strip final "/" if it exists
- if (prefix.endsWith("/")) {
- this.prefix = prefix.substring(0, prefix.length() - 1);
- } else {
- this.prefix = prefix;
- }
- }
-
- /**
- * If you want to customize the output file's file extension.
- * Do not include the "."
- *
- * @param fileExtension
- */
- @Field
- public void setFileExtension(String fileExtension) {
- this.fileExtension = fileExtension;
- }
-
-
- /**
- * This initializes the az blob container client
- *
- * @param params params to use for initialization
- * @throws TikaConfigException
- */
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- //TODO -- allow authentication via other methods
- blobServiceClient = new BlobServiceClientBuilder()
- .endpoint(endpoint)
- .credential(new AzureSasCredential(sasToken))
- .buildClient();
- blobContainerClient = blobServiceClient.getBlobContainerClient(container);
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
- mustNotBeEmpty("sasToken", this.sasToken);
- mustNotBeEmpty("endpoint", this.endpoint);
- mustNotBeEmpty("container", this.container);
- }
-
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/java/org/apache/tika/pipes/emitter/azblob/TestAZBlobEmitter.java b/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/java/org/apache/tika/pipes/emitter/azblob/TestAZBlobEmitter.java
deleted file mode 100644
index bed2367a0..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/java/org/apache/tika/pipes/emitter/azblob/TestAZBlobEmitter.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.azblob;
-
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.Emitter;
-import org.apache.tika.pipes.core.emitter.EmitterManager;
-
-@Disabled("turn into an actual test")
-public class TestAZBlobEmitter {
-
- @Test
- public void testBasic() throws Exception {
- EmitterManager emitterManager = EmitterManager.load(getConfig("tika-config-az-blob.xml"));
- Emitter emitter = emitterManager.getEmitter("az-blob");
- List<Metadata> metadataList = new ArrayList<>();
- Metadata m = new Metadata();
- m.set("k1", "v1");
- m.add("k1", "v2");
- m.set("k2", "v3");
- m.add("k2", "v4");
- metadataList.add(m);
- emitter.emit("something-or-other/test-out", metadataList, new ParseContext());
- }
-
- private Path getConfig(String configFile) throws URISyntaxException {
- return Paths.get(this
- .getClass()
- .getResource("/config/" + configFile)
- .toURI());
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/resources/config/tika-config-az-blob.xml b/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/resources/config/tika-config-az-blob.xml
deleted file mode 100644
index c1ba42b07..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/resources/config/tika-config-az-blob.xml
+++ /dev/null
@@ -1,28 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.azblob.AZBlobEmitter">
- <name>az-blob</name>
- <!-- these have to be non-null -->
- <endpoint></endpoint>
- <container></container>
- <sasToken></sasToken>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-gcs/pom.xml b/tika-pipes/tika-emitters/tika-emitter-gcs/pom.xml
deleted file mode 100644
index afdba7ace..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-gcs/pom.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-gcs</artifactId>
- <name>Apache Tika GCS emitter</name>
-
-
- <dependencies>
- <!-- should serialization be provided or bundled? -->
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-serialization</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.cloud</groupId>
- <artifactId>google-cloud-storage</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.emitter.gcs</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java b/tika-pipes/tika-emitters/tika-emitter-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java
deleted file mode 100644
index fd0f0ce7c..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.gcs;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.List;
-import java.util.Map;
-
-import com.google.cloud.storage.BlobId;
-import com.google.cloud.storage.BlobInfo;
-import com.google.cloud.storage.Storage;
-import com.google.cloud.storage.StorageOptions;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.StreamEmitter;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-import org.apache.tika.serialization.JsonMetadataList;
-import org.apache.tika.utils.StringUtils;
-
-
-public class GCSEmitter extends AbstractEmitter implements Initializable, StreamEmitter {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GCSEmitter.class);
- private String projectId;
- private String bucket;
- private String fileExtension = "json";
- private String prefix = null;
- private Storage storage;
-
- /**
- * Requires the src-bucket/path/to/my/file.txt in the {@link TikaCoreProperties#SOURCE_PATH}.
- *
- * @param metadataList
- * @throws IOException
- * @throws TikaException
- */
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext) throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.size() == 0) {
- throw new TikaEmitterException("metadata list must not be null or of size 0");
- }
- try (UnsynchronizedByteArrayOutputStream bos = UnsynchronizedByteArrayOutputStream
- .builder()
- .get()) {
- try (Writer writer = new OutputStreamWriter(bos, StandardCharsets.UTF_8)) {
- JsonMetadataList.toJson(metadataList, writer);
- } catch (IOException e) {
- throw new TikaEmitterException("can't jsonify", e);
- }
-
- write(emitKey, new Metadata(), bos.toByteArray());
- }
-
- }
-
- /**
- * @param path -- object path, not including the bucket
- * @param is inputStream to copy
- * @param userMetadata this will be written to the s3 ObjectMetadata's userMetadata
- * @throws TikaEmitterException or IOexception if there is a Runtime s3 client exception
- */
- @Override
- public void emit(String path, InputStream is, Metadata userMetadata, ParseContext parseContext) throws IOException, TikaEmitterException {
-
- if (is instanceof TikaInputStream && ((TikaInputStream) is).hasFile()) {
- write(path, userMetadata, Files.readAllBytes(((TikaInputStream) is).getPath()));
- } else {
- try (UnsynchronizedByteArrayOutputStream bos = UnsynchronizedByteArrayOutputStream
- .builder()
- .get()) {
- IOUtils.copy(is, bos);
- write(path, userMetadata, bos.toByteArray());
- }
- }
- }
-
- private void write(String path, Metadata userMetadata, byte[] bytes) {
- if (!StringUtils.isBlank(prefix)) {
- path = prefix + "/" + path;
- }
-
- if (!StringUtils.isBlank(fileExtension)) {
- path += "." + fileExtension;
- }
-
- LOGGER.debug("about to emit to target bucket: ({}) path:({})", bucket, path);
- BlobId blobId = BlobId.of(bucket, path);
- BlobInfo blobInfo = BlobInfo
- .newBuilder(blobId)
- .build();
-
- for (String n : userMetadata.names()) {
- String[] vals = userMetadata.getValues(n);
- if (vals.length > 1) {
- LOGGER.warn("Can only write the first value for key {}. I see {} values.", n, vals.length);
- }
- blobInfo
- .getMetadata()
- .put(n, vals[0]);
- }
- storage.create(blobInfo, bytes);
- }
-
-
- @Field
- public void setProjectId(String projectId) {
- this.projectId = projectId;
- }
-
- @Field
- public void setBucket(String bucket) {
- this.bucket = bucket;
- }
-
- @Field
- public void setPrefix(String prefix) {
- //strip final "/" if it exists
- if (prefix.endsWith("/")) {
- this.prefix = prefix.substring(0, prefix.length() - 1);
- } else {
- this.prefix = prefix;
- }
- }
-
- /**
- * If you want to customize the output file's file extension.
- * Do not include the "."
- *
- * @param fileExtension
- */
- @Field
- public void setFileExtension(String fileExtension) {
- this.fileExtension = fileExtension;
- }
-
-
- /**
- * This initializes the gcs client.
- *
- * @param params params to use for initialization
- * @throws TikaConfigException
- */
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- //params have already been set...ignore them
- //TODO -- add other params to the builder as needed
- storage = StorageOptions
- .newBuilder()
- .setProjectId(projectId)
- .build()
- .getService();
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
- mustNotBeEmpty("bucket", this.bucket);
- mustNotBeEmpty("projectId", this.projectId);
- }
-
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/java/org/apache/tika/pipes/emitter/gcs/TestGCSEmitter.java b/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/java/org/apache/tika/pipes/emitter/gcs/TestGCSEmitter.java
deleted file mode 100644
index 23c42aa0f..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/java/org/apache/tika/pipes/emitter/gcs/TestGCSEmitter.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.gcs;
-
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.Emitter;
-import org.apache.tika.pipes.core.emitter.EmitterManager;
-
-@Disabled("turn into an actual test")
-public class TestGCSEmitter {
-
- @Test
- public void testBasic() throws Exception {
- EmitterManager emitterManager = EmitterManager.load(getConfig("tika-config-gcs.xml"));
- Emitter emitter = emitterManager.getEmitter("gcs");
- List<Metadata> metadataList = new ArrayList<>();
- Metadata m = new Metadata();
- m.set("k1", "v1");
- m.add("k1", "v2");
- m.set("k2", "v3");
- metadataList.add(m);
- emitter.emit("something-or-other/test-out", metadataList, new ParseContext());
- }
-
- private Path getConfig(String configFile) throws URISyntaxException {
- return Paths.get(this
- .getClass()
- .getResource("/config/" + configFile)
- .toURI());
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/resources/config/tika-config-gcs.xml b/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/resources/config/tika-config-gcs.xml
deleted file mode 100644
index 2ea06761e..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/resources/config/tika-config-gcs.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.gcs.GCSEmitter">
- <name>gcs</name>
- <projectId>My First Project</projectId>
- <bucket>tika-tallison-test-bucket</bucket>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/pom.xml b/tika-pipes/tika-emitters/tika-emitter-jdbc/pom.xml
deleted file mode 100644
index 6698b624d..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/pom.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-jdbc</artifactId>
- <name>Apache Tika jdbc emitter</name>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.emitter.jdbc</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/main/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitter.java b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/main/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitter.java
deleted file mode 100644
index ced0a92ff..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/main/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitter.java
+++ /dev/null
@@ -1,701 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.jdbc;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.sql.Timestamp;
-import java.sql.Types;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.EmitData;
-import org.apache.tika.pipes.core.emitter.EmitKey;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-import org.apache.tika.utils.StringUtils;
-
-/**
- * This is only an initial, basic implementation of an emitter for JDBC.
- * <p>
- * It is currently NOT thread safe because of the shared prepared statement,
- * and depending on the jdbc implementation because of the shared connection.
- * <p>
- * As of the 2.5.0 release, this is ALPHA version. There may be breaking changes
- * in the future.
- */
-public class JDBCEmitter extends AbstractEmitter implements Initializable, Closeable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(JDBCEmitter.class);
-
- public enum AttachmentStrategy {
- FIRST_ONLY, ALL
- //anything else?
- }
-
- public enum MultivaluedFieldStrategy {
- FIRST_ONLY, CONCATENATE
- //anything else?
- }
-
- //some file formats do not have time zones...
- //try both
- private static final String[] TIKA_DATE_PATTERNS =
- new String[] {"yyyy-MM-dd'T'HH:mm:ss'Z'", "yyyy-MM-dd'T'HH:mm:ss"};
- //the "write" lock is used for creating the table
- private static ReadWriteLock READ_WRITE_LOCK = new ReentrantReadWriteLock();
- //this keeps track of which table + connection string have been created
- //so that only one table is created per table + connection string.
- //This is necessary for testing and if someone specifies multiple
- //different jdbc emitters.
- private static Set<String> TABLES_CREATED = new HashSet<>();
- private String connectionString;
-
- private Optional<String> postConnectionString = Optional.empty();
- private String insert;
- private String createTable;
- private String alterTable;
-
- private int maxRetries = 0;
-
- //used only for specification of column name/string definition of
- //keys
- private Map<String, String> keys;
-
- private List<ColumnDefinition> columns;
- private Connection connection;
- private PreparedStatement insertStatement;
- private AttachmentStrategy attachmentStrategy = AttachmentStrategy.FIRST_ONLY;
-
- private MultivaluedFieldStrategy multivaluedFieldStrategy =
- MultivaluedFieldStrategy.CONCATENATE;
-
- private String multivaluedFieldDelimiter = ", ";
-
- //emitters are run in a single thread. If we ever start running them
- //multithreaded, this will be a big problem.
- private final DateFormat[] dateFormats;
-
- private int maxStringLength = 64000;
-
- //this is set during the initialize phase
- private StringNormalizer stringNormalizer;
-
- public JDBCEmitter() {
- dateFormats = new DateFormat[TIKA_DATE_PATTERNS.length];
- int i = 0;
- for (String p : TIKA_DATE_PATTERNS) {
- dateFormats[i++] = new SimpleDateFormat(p, Locale.US);
- }
- }
-
- /**
- * This is called immediately after the table is created.
- * The purpose of this is to allow for adding a complex primary key or
- * other constraint on the table after it is created.
- *
- * @param alterTable
- */
- public void setAlterTable(String alterTable) {
- this.alterTable = alterTable;
- }
-
- @Field
- public void setCreateTable(String createTable) {
- this.createTable = createTable;
- }
-
- @Field
- public void setInsert(String insert) {
- this.insert = insert;
- }
-
- @Field
- public void setConnection(String connection) {
- this.connectionString = connection;
- }
-
- /**
- * Set the maximum string length in characters (not bytes).
- * This is applies only to fields with name "string"
- * not to "varchar".
- *
- * @param maxStringLength
- */
- @Field
- public void setMaxStringLength(int maxStringLength) {
- this.maxStringLength = maxStringLength;
- }
-
- public void setMaxRetries(int maxRetries) {
- this.maxRetries = maxRetries;
- }
-
- /**
- * This sql will be called immediately after the connection is made. This was
- * initially added for setting pragmas on sqlite3, but may be used for other
- * connection configuration in other dbs. Note: This is called before the table is
- * created if it needs to be created.
- *
- * @param postConnection
- */
- @Field
- public void setPostConnection(String postConnection) {
- this.postConnectionString = Optional.of(postConnection);
- }
-
- /**
- * This applies to fields of type 'string' or 'varchar'. If there's
- * a multivalued field in a metadata object, do you want the first value only
- * or should we concatenate these with the
- * {@link JDBCEmitter#setMultivaluedFieldDelimiter(String)}.
- * <p>
- * The default values as of 2.6.1 are {@link MultivaluedFieldStrategy#CONCATENATE}
- * and the default delimiter is ", "
- *
- * @param strategy
- * @throws TikaConfigException
- */
- @Field
- public void setMultivaluedFieldStrategy(String strategy) throws TikaConfigException {
- String lc = strategy.toLowerCase(Locale.US);
- if (lc.equals("first_only")) {
- setMultivaluedFieldStrategy(MultivaluedFieldStrategy.FIRST_ONLY);
- } else if (lc.equals("concatenate")) {
- setMultivaluedFieldStrategy(MultivaluedFieldStrategy.CONCATENATE);
- } else {
- throw new TikaConfigException("I'm sorry, I only recogize 'first_only' and " +
- "'concatenate'. I don't mind '" + strategy + "'");
- }
- }
-
- public void setMultivaluedFieldStrategy(MultivaluedFieldStrategy multivaluedFieldStrategy) {
- this.multivaluedFieldStrategy = multivaluedFieldStrategy;
- }
-
- /**
- * See {@link JDBCEmitter#setMultivaluedFieldDelimiter(String)}
- *
- * @param delimiter
- */
- @Field
- public void setMultivaluedFieldDelimiter(String delimiter) {
- this.multivaluedFieldDelimiter = delimiter;
- }
-
- /**
- * The implementation of keys should be a LinkedHashMap because
- * order matters!
- * <p>
- * Key is the name of the metadata field, value is the type of column:
- * boolean, string, int, long
- *
- * @param keys
- */
- @Field
- public void setKeys(Map<String, String> keys) {
- this.keys = keys;
- }
-
- public void setAttachmentStrategy(AttachmentStrategy attachmentStrategy) {
- this.attachmentStrategy = attachmentStrategy;
- }
-
- @Field
- public void setAttachmentStrategy(String attachmentStrategy) {
- if ("all".equalsIgnoreCase(attachmentStrategy)) {
- setAttachmentStrategy(AttachmentStrategy.ALL);
- } else if ("first_only".equalsIgnoreCase(attachmentStrategy)) {
- setAttachmentStrategy(AttachmentStrategy.FIRST_ONLY);
- } else {
- throw new IllegalArgumentException("attachmentStrategy must be 'all' or 'first_only'");
- }
- }
-
- /**
- * This executes the emit with each call. For more efficient
- * batch execution use {@link #emit(List)}.
- *
- * @param emitKey emit key
- * @param metadataList list of metadata per file
- * @throws IOException
- * @throws TikaEmitterException
- */
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext)
- throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.size() < 1) {
- return;
- }
- List<EmitData> emitDataList = new ArrayList<>();
- emitDataList.add(new EmitData(new EmitKey("", emitKey), metadataList));
- emit(emitDataList);
- }
-
- @Override
- public void emit(List<? extends EmitData> emitData) throws IOException, TikaEmitterException {
- int tries = 0;
- Exception ex = null;
- while (tries++ <= maxRetries) {
- try {
- emitNow(emitData);
- return;
- } catch (SQLException e) {
- try {
- reconnect();
- } catch (SQLException exc) {
- throw new TikaEmitterException("couldn't reconnect!", exc);
- }
- ex = e;
- }
- }
- throw new TikaEmitterException("Couldn't emit " + emitData.size() + " records.", ex);
- }
-
- private void emitNow(List<? extends EmitData> emitData) throws SQLException {
- if (attachmentStrategy == AttachmentStrategy.FIRST_ONLY) {
- for (EmitData d : emitData) {
- insertFirstOnly(d.getEmitKey().getEmitKey(), d.getMetadataList());
- insertStatement.addBatch();
- }
- } else {
- for (EmitData d : emitData) {
- insertAll(d.getEmitKey().getEmitKey(), d.getMetadataList());
- }
- }
- if (LOGGER.isDebugEnabled()) {
- long start = System.currentTimeMillis();
- insertStatement.executeBatch();
- connection.commit();
- LOGGER.debug("took {}ms to insert {} rows ", System.currentTimeMillis() - start,
- emitData.size());
- } else {
- insertStatement.executeBatch();
- connection.commit();
- }
-
- }
-
- private void insertAll(String emitKey, List<Metadata> metadataList) throws SQLException {
-
- for (int i = 0; i < metadataList.size(); i++) {
- insertStatement.clearParameters();
- int col = 0;
- insertStatement.setString(++col, emitKey);
- insertStatement.setInt(++col, i);
- for (ColumnDefinition columnDefinition : columns) {
- updateValue(emitKey, insertStatement, ++col, columnDefinition, i, metadataList);
- }
- insertStatement.addBatch();
- }
- }
-
- private void insertFirstOnly(String emitKey, List<Metadata> metadataList) throws SQLException {
- insertStatement.clearParameters();
- int i = 0;
- insertStatement.setString(++i, emitKey);
- for (ColumnDefinition columnDefinition : columns) {
- updateValue(emitKey, insertStatement, ++i, columnDefinition, 0, metadataList);
- }
- }
-
- private void reconnect() throws SQLException {
- SQLException ex = null;
- for (int i = 0; i < 3; i++) {
- try {
- tryClose();
- createConnection();
- insertStatement = connection.prepareStatement(insert);
- return;
- } catch (SQLException e) {
- LOGGER.warn("couldn't reconnect to db", e);
- ex = e;
- }
- }
- throw ex;
- }
-
- private void tryClose() {
- if (insertStatement != null) {
- try {
- insertStatement.close();
- } catch (SQLException e) {
- LOGGER.warn("exception closing insert", e);
- }
- }
-
- if (connection != null) {
- try {
- connection.commit();
- connection.close();
- } catch (SQLException e) {
- LOGGER.warn("exception closing connection", e);
- }
- }
- }
-
- private void createConnection() throws SQLException {
- connection = DriverManager.getConnection(connectionString);
- connection.setAutoCommit(false);
- if (postConnectionString.isPresent()) {
- try (Statement st = connection.createStatement()) {
- st.execute(postConnectionString.get());
- }
- }
- }
-
- private void updateValue(String emitKey, PreparedStatement insertStatement, int i,
- ColumnDefinition columnDefinition, int metadataListIndex,
- List<Metadata> metadataList)
- throws SQLException {
- Metadata metadata = metadataList.get(metadataListIndex);
- String val = getVal(metadata, columnDefinition);
- switch (columnDefinition.getType()) {
- case Types.VARCHAR:
- updateVarchar(emitKey, columnDefinition, insertStatement, i, val);
- break;
- case Types.BOOLEAN:
- updateBoolean(insertStatement, i, val);
- break;
- case Types.INTEGER:
- updateInteger(insertStatement, i, val);
- break;
- case Types.BIGINT:
- updateLong(insertStatement, i, val);
- break;
- case Types.FLOAT:
- updateFloat(insertStatement, i, val);
- break;
- case Types.DOUBLE:
- updateDouble(insertStatement, i, val);
- break;
- case Types.TIMESTAMP:
- updateTimestamp(insertStatement, i, val, dateFormats);
- break;
- default:
- throw new IllegalArgumentException(
- "Can only process:" + getHandledTypes() +
- " types so far. " +
- "Please open a ticket to request: " +
- columnDefinition.getType() + " for " +
- columnDefinition.getColumnName());
- }
- }
-
- private String getVal(Metadata metadata, ColumnDefinition columnDefinition) {
- if (columnDefinition.getType() != Types.VARCHAR) {
- return metadata.get(columnDefinition.getColumnName());
- }
- if (multivaluedFieldStrategy == MultivaluedFieldStrategy.FIRST_ONLY) {
- return metadata.get(columnDefinition.getColumnName());
- }
- String[] vals = metadata.getValues(columnDefinition.getColumnName());
- if (vals.length == 0) {
- return null;
- } else if (vals.length == 1) {
- return vals[0];
- }
-
- int i = 0;
- StringBuilder sb = new StringBuilder();
- for (String val : metadata.getValues(columnDefinition.getColumnName())) {
- if (StringUtils.isBlank(val)) {
- continue;
- }
- if (i > 0) {
- sb.append(multivaluedFieldDelimiter);
- }
- sb.append(val);
- i++;
- }
- return sb.toString();
- }
-
- private void updateDouble(PreparedStatement insertStatement, int i, String val)
- throws SQLException {
- if (StringUtils.isBlank(val)) {
- insertStatement.setNull(i, Types.DOUBLE);
- return;
- }
- Double d = Double.parseDouble(val);
- insertStatement.setDouble(i, d);
- }
-
- private void updateVarchar(String emitKey, ColumnDefinition columnDefinition, PreparedStatement insertStatement,
- int i,
- String val) throws SQLException {
- if (val == null) {
- insertStatement.setNull(i, Types.VARCHAR);
- return;
- }
- String normalized = stringNormalizer.normalize(emitKey,
- columnDefinition.getColumnName(), val, columnDefinition.getPrecision());
- insertStatement.setString(i, normalized);
- }
-
- private void updateTimestamp(PreparedStatement insertStatement, int i, String val,
- DateFormat[] dateFormats) throws SQLException {
- if (StringUtils.isBlank(val)) {
- insertStatement.setNull(i, Types.TIMESTAMP);
- return;
- }
-
- for (DateFormat df : dateFormats) {
- try {
- Date d = df.parse(val);
- insertStatement.setTimestamp(i, new Timestamp(d.getTime()));
- return;
- } catch (ParseException e) {
- //ignore
- }
- }
- LOGGER.warn("Couldn't parse {}" + val);
- insertStatement.setNull(i, Types.TIMESTAMP);
- }
-
- private void updateFloat(PreparedStatement insertStatement, int i, String val)
- throws SQLException {
- if (StringUtils.isBlank(val)) {
- insertStatement.setNull(i, Types.FLOAT);
- } else {
- insertStatement.setFloat(i, Float.parseFloat(val));
- }
- }
-
- private void updateLong(PreparedStatement insertStatement, int i, String val)
- throws SQLException {
- if (StringUtils.isBlank(val)) {
- insertStatement.setNull(i, Types.BIGINT);
- } else {
- insertStatement.setLong(i, Long.parseLong(val));
- }
- }
-
- private void updateInteger(PreparedStatement insertStatement, int i, String val)
- throws SQLException {
- if (StringUtils.isBlank(val)) {
- insertStatement.setNull(i, Types.INTEGER);
- } else {
- insertStatement.setInt(i, Integer.parseInt(val));
- }
- }
-
- private void updateBoolean(PreparedStatement insertStatement, int i, String val)
- throws SQLException {
- if (StringUtils.isBlank(val)) {
- insertStatement.setNull(i, Types.BOOLEAN);
- } else {
- insertStatement.setBoolean(i, Boolean.parseBoolean(val));
- }
- }
-
-
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- parseColTypes();
- setStringNormalizer();
- try {
- createConnection();
- } catch (SQLException e) {
- throw new TikaConfigException("couldn't open connection: " + connectionString, e);
- }
- if (!StringUtils.isBlank(createTable)) {
- //synchronize table creation
- READ_WRITE_LOCK.writeLock().lock();
- try {
- String tableCreationString = connectionString + " " + createTable;
- if (!TABLES_CREATED.contains(tableCreationString)) {
- try (Statement st = connection.createStatement()) {
- st.execute(createTable);
- if (!StringUtils.isBlank(alterTable)) {
- st.execute(alterTable);
- }
- TABLES_CREATED.add(tableCreationString);
- } catch (SQLException e) {
- throw new TikaConfigException("can't create table", e);
- }
- }
- } finally {
- READ_WRITE_LOCK.writeLock().unlock();
- }
- }
- try {
- insertStatement = connection.prepareStatement(insert);
- } catch (SQLException e) {
- throw new TikaConfigException("can't create insert statement", e);
- }
- }
-
- private void setStringNormalizer() {
- if (connectionString.startsWith("jdbc:postgres")) {
- stringNormalizer = new JDBCEmitter.PostgresNormalizer();
- } else {
- stringNormalizer = new JDBCEmitter.StringNormalizer();
- }
- }
-
- private void parseColTypes() {
- columns = new ArrayList<>();
- for (Map.Entry<String, String> e : keys.entrySet()) {
- columns.add(ColumnDefinition.parse(e.getKey(), e.getValue(), maxStringLength));
- }
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler)
- throws TikaConfigException {
- //require
- }
-
- /**
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- try {
- insertStatement.close();
- } catch (SQLException e) {
- LOGGER.warn("problem closing insert", e);
- }
- try {
- connection.close();
- } catch (SQLException e) {
- throw new IOException(e);
- }
- }
-
- private static String getHandledTypes() {
- return "'string', 'varchar', " +
- "'boolean', 'int', 'long', 'float', 'double' and 'timestamp'";
- }
-
- private static class StringNormalizer {
-
- String normalize(String emitKey, String columnName, String s, int maxLength) {
- if (maxLength < 0 || s.length() <= maxLength) {
- return s;
- }
- LOGGER.warn("truncating {}->'{}' from {} chars to {} chars",
- emitKey, columnName, s.length(), maxLength);
-
- return s.substring(0, maxLength);
- }
- }
-
- private static class PostgresNormalizer extends StringNormalizer {
-
- @Override
- String normalize(String emitKey, String columnName, String s, int maxLength) {
- s = s.replaceAll("\u0000", " ");
- return super.normalize(emitKey, columnName, s, maxLength);
- }
- }
-
- private static class ColumnDefinition {
- private static final Matcher VARCHAR_MATCHER =
- Pattern.compile("varchar\\((\\d+)\\)").matcher("");
-
- private final String columnName;
-
- private final int type;
- //this is only used (so far) for varchar. It is currently
- //ignored for other data types
- private final int precision;
-
- private static ColumnDefinition parse(String name, String type, int maxStringLength) {
- String lcType = type.toLowerCase(Locale.US);
- if (VARCHAR_MATCHER.reset(lcType).find()) {
- return new ColumnDefinition(name,
- Types.VARCHAR, Integer.parseInt(VARCHAR_MATCHER.group(1)));
- }
-
- switch (lcType) {
-
- case "string":
- return new ColumnDefinition(name, Types.VARCHAR, maxStringLength);
- case "bool":
- case "boolean":
- return new ColumnDefinition(name, Types.BOOLEAN, -1);
- case "int":
- case "integer":
- return new ColumnDefinition(name, Types.INTEGER, -1);
- case "bigint":
- case "long":
- return new ColumnDefinition(name, Types.BIGINT, -1);
- case "float":
- return new ColumnDefinition(name, Types.FLOAT, -1);
- case "double":
- return new ColumnDefinition(name, Types.DOUBLE, -1);
- case "timestamp":
- return new ColumnDefinition(name, Types.TIMESTAMP, -1);
-
- default:
- throw new IllegalArgumentException(
- "Can only process: " + getHandledTypes() +
- " types so far. Please open a ticket to request " +
- type + " for column: " + name);
- }
- }
-
- private ColumnDefinition(String columnName, int type, int precision) {
- this.columnName = columnName;
- this.type = type;
- this.precision = precision;
- }
-
- public String getColumnName() {
- return columnName;
- }
-
- public int getType() {
- return type;
- }
-
- public int getPrecision() {
- return precision;
- }
- }
-
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java
deleted file mode 100644
index 448484197..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.jdbc;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.Statement;
-import java.sql.Timestamp;
-import java.time.ZoneId;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.Emitter;
-import org.apache.tika.pipes.core.emitter.EmitterManager;
-
-public class JDBCEmitterTest {
-
- @Test
- public void testBasic(@TempDir Path tmpDir) throws Exception {
- Files.createDirectories(tmpDir.resolve("db"));
- Path dbDir = tmpDir.resolve("db/h2");
- Path config = tmpDir.resolve("tika-config.xml");
- String connectionString = "jdbc:h2:file:" + dbDir.toAbsolutePath();
-
- writeConfig("/configs/tika-config-jdbc-emitter.xml",
- connectionString, config);
-
- EmitterManager emitterManager = EmitterManager.load(config);
- Emitter emitter = emitterManager.getEmitter();
- List<String[]> data = new ArrayList<>();
- data.add(new String[]{"k1", "true", "k2", "some string1", "k3", "4", "k4", "100"});
- data.add(new String[]{"k1", "false", "k2", "some string2", "k3", "5", "k4", "101"});
- data.add(new String[]{"k1", "true", "k2", "some string3", "k3", "6", "k4", "102"});
- //test dates with and without timezones
- data.add(new String[]{"k1", "false", "k2", "some string4", "k3", "7", "k4", "103", "k5",
- "100002", "k6", "2022-11-04T17:10:15Z"});
-
- data.add(new String[]{"k1", "true", "k2", "some string5", "k3", "8", "k4", "104", "k5",
- "100002", "k6", "2022-11-04T17:10:15"});
- int id = 0;
- for (String[] d : data) {
- emitter.emit("id" + id++, Collections.singletonList(m(d)), new ParseContext());
- }
-
- try (Connection connection = DriverManager.getConnection(connectionString)) {
- try (Statement st = connection.createStatement()) {
- try (ResultSet rs = st.executeQuery("select * from test")) {
- int rows = 0;
- while (rs.next()) {
- assertEquals("id" + rows, rs.getString(1));
- assertEquals(rows % 2 == 0, rs.getBoolean(2));
- assertEquals("some string" + (rows + 1), rs.getString(3));
- assertEquals(rows + 4, rs.getInt(4));
- assertEquals(100 + rows, rs.getLong(5));
- if (rows > 2) {
- assertEquals(100002, rs.getLong(6));
- Timestamp timestamp = rs.getTimestamp(7);
- String str = timestamp.toInstant().atZone(ZoneId.of("UTC")).toString();
- //TODO fix this to work in other timezones
- assertTrue(str.startsWith("2022-11"));
- }
- rows++;
- }
- }
- }
- }
- }
-
- @Test
- public void testTableExists(@TempDir Path tmpDir) throws Exception {
- String createTable = "create table test (path varchar(512) primary key," +
- "k1 boolean,k2 varchar(512),k3 integer,k4 long);";
-
- Files.createDirectories(tmpDir.resolve("db"));
- Path dbDir = tmpDir.resolve("db/h2");
- Path config = tmpDir.resolve("tika-config.xml");
- String connectionString = "jdbc:h2:file:" + dbDir.toAbsolutePath();
- writeConfig("/configs/tika-config-jdbc-emitter-existing-table.xml",
- connectionString, config);
-
- try (Connection connection = DriverManager.getConnection(connectionString)) {
- connection.createStatement().execute(createTable);
- }
- EmitterManager emitterManager = EmitterManager.load(config);
- Emitter emitter = emitterManager.getEmitter();
- List<String[]> data = new ArrayList<>();
- data.add(new String[]{"k1", "true", "k2", "some string1", "k3", "4", "k4", "100"});
- data.add(new String[]{"k1", "false", "k2", "some string2", "k3", "5", "k4", "101"});
- data.add(new String[]{"k1", "true", "k2", "some string3", "k3", "6", "k4", "102"});
- int id = 0;
- for (String[] d : data) {
- emitter.emit("id" + id++, Collections.singletonList(m(d)), new ParseContext());
- }
-
- try (Connection connection = DriverManager.getConnection(connectionString)) {
- try (Statement st = connection.createStatement()) {
- try (ResultSet rs = st.executeQuery("select * from test")) {
- int rows = 0;
- while (rs.next()) {
- assertEquals("id" + rows, rs.getString(1));
- assertEquals(rows % 2 == 0, rs.getBoolean(2));
- assertEquals("some string" + (rows + 1), rs.getString(3));
- assertEquals(rows + 4, rs.getInt(4));
- assertEquals(100 + rows, rs.getLong(5));
- rows++;
- }
- }
- }
- }
- }
-
- @Test
- public void testAttachments(@TempDir Path tmpDir) throws Exception {
- Files.createDirectories(tmpDir.resolve("db"));
- Path dbDir = tmpDir.resolve("db/h2");
- Path config = tmpDir.resolve("tika-config.xml");
- String connectionString = "jdbc:h2:file:" + dbDir.toAbsolutePath();
-
- writeConfig("/configs/tika-config-jdbc-emitter-attachments.xml",
- connectionString, config);
-
- EmitterManager emitterManager = EmitterManager.load(config);
- Emitter emitter = emitterManager.getEmitter();
- List<Metadata> data = new ArrayList<>();
- data.add(m("k1", "true", "k2", "some string1", "k3", "4", "k4", "100"));
- data.add(m("k1", "false", "k2", "some string2", "k3", "5", "k4", "101"));
- data.add(m("k1", "true", "k2", "some string3", "k3", "6", "k4", "102"));
- emitter.emit("id0", data, new ParseContext());
-
-
- try (Connection connection = DriverManager.getConnection(connectionString)) {
- try (Statement st = connection.createStatement()) {
- try (ResultSet rs = st.executeQuery("select * from test")) {
- int rows = 0;
- assertEquals("path", rs.getMetaData().getColumnName(1).toLowerCase(Locale.US));
- assertEquals("attachment_num",
- rs.getMetaData().getColumnName(2).toLowerCase(Locale.US));
- while (rs.next()) {
- assertEquals("id0", rs.getString(1));
- assertEquals(rows, rs.getInt(2));
- assertEquals(rows % 2 == 0, rs.getBoolean(3));
- assertEquals("some string" + (rows + 1), rs.getString(4));
- assertEquals(rows + 4, rs.getInt(5));
- assertEquals(100 + rows, rs.getLong(6));
- rows++;
- }
- }
- }
- }
- }
-
- @Test
- public void testMultiValuedFields(@TempDir Path tmpDir) throws Exception {
- Files.createDirectories(tmpDir.resolve("db"));
- Path dbDir = tmpDir.resolve("db/h2");
- Path config = tmpDir.resolve("tika-config.xml");
- String connectionString = "jdbc:h2:file:" + dbDir.toAbsolutePath();
-
- writeConfig("/configs/tika-config-jdbc-emitter-multivalued.xml",
- connectionString, config);
-
- EmitterManager emitterManager = EmitterManager.load(config);
- Emitter emitter = emitterManager.getEmitter();
- List<Metadata> data = new ArrayList<>();
- Metadata m = new Metadata();
- m.add("k1", "first");
- m.add("k1", "second");
- m.add("k1", "third");
- m.add("k1", "fourth");
- data.add(m);
- emitter.emit("id0", data, new ParseContext());
-
- String expected = "first, second, third, fourth";
- int rows = 0;
- try (Connection connection = DriverManager.getConnection(connectionString)) {
- try (Statement st = connection.createStatement()) {
- try (ResultSet rs = st.executeQuery("select * from test")) {
- assertEquals("path", rs.getMetaData().getColumnName(1).toLowerCase(Locale.US));
- while (rs.next()) {
- assertEquals("id0", rs.getString(1));
- assertEquals(expected, rs.getString(2));
- rows++;
- }
- }
- }
- }
- assertEquals(1, rows);
- }
-
- @Test
- public void testVarcharTruncation(@TempDir Path tmpDir) throws Exception {
- Files.createDirectories(tmpDir.resolve("db"));
- Path dbDir = tmpDir.resolve("db/h2");
- Path config = tmpDir.resolve("tika-config.xml");
- String connectionString = "jdbc:h2:file:" + dbDir.toAbsolutePath();
-
- writeConfig("/configs/tika-config-jdbc-emitter-trunc.xml",
- connectionString, config);
-
- EmitterManager emitterManager = EmitterManager.load(config);
- Emitter emitter = emitterManager.getEmitter();
- List<String[]> data = new ArrayList<>();
- data.add(new String[]{"k1", "abcd"});
- data.add(new String[]{"k1", "abcdefghijklmnopqrs"});
- data.add(new String[]{"k1", "abcdefghijk"});
- int id = 0;
- for (String[] d : data) {
- emitter.emit("id" + id++, Collections.singletonList(m(d)), new ParseContext());
- }
-
- int rows = 0;
- try (Connection connection = DriverManager.getConnection(connectionString)) {
- try (Statement st = connection.createStatement()) {
- try (ResultSet rs = st.executeQuery("select * from test")) {
- while (rs.next()) {
- String s = rs.getString(2);
- assertTrue(s.length() < 13);
- assertFalse(s.contains("m"));
- rows++;
- }
- }
- }
- }
- assertEquals(3, rows);
- }
-
- private void writeConfig(String srcConfig, String dbDir, Path config) throws IOException {
- String xml = IOUtils.resourceToString(srcConfig, StandardCharsets.UTF_8);
- xml = xml.replace("CONNECTION_STRING", dbDir);
- Files.write(config, xml.getBytes(StandardCharsets.UTF_8));
- }
-
- private Metadata m(String... strings) {
- Metadata metadata = new Metadata();
- for (int i = 0; i < strings.length; i++) {
- metadata.set(strings[i], strings[++i]);
- }
- return metadata;
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-attachments.xml b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-attachments.xml
deleted file mode 100644
index 4bc2d8e87..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-attachments.xml
+++ /dev/null
@@ -1,53 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.jdbc.JDBCEmitter">
- <name>jdbc</name>
- <connection>CONNECTION_STRING</connection>
- <createTable>create table test
- (path varchar(512) not null,
- attachment_num integer not null,
- k1 boolean,
- k2 varchar(512),
- k3 integer,
- k4 long);
- </createTable>
- <alterTable>alter table test add primary key (path, attachment_num)</alterTable>
- <!-- the jdbc emitter always puts ths emitKey value as the first
- item -->
- <insert>insert into test (path, attachment_num, k1, k2, k3, k4) values (?,?,?,?,?,?);
- </insert>
- <!-- these are the keys in the metadata object.
- The emitKey is added as the first element in the insert statement.
- Then the these values are added in order.
- They must be in the order of the insert statement.
- The emit key is added as
- -->
- <keys>
- <key k="k1" v="boolean"/>
- <key k="k2" v="string"/>
- <key k="k3" v="int"/>
- <key k="k4" v="long"/>
- </keys>
- <attachmentStrategy>all</attachmentStrategy>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-existing-table.xml b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-existing-table.xml
deleted file mode 100644
index 654b279be..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-existing-table.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.jdbc.JDBCEmitter">
- <name>jdbc</name>
- <connection>CONNECTION_STRING</connection>
- <insert>insert into test (path, k1, k2, k3, k4) values (?,?,?,?,?);
- </insert>
- <!-- these are the keys in the metadata object.
- The emitKey is added as the first element in the insert statement.
- Then the these values are added in order.
- They must be in the order of the insert statement.
- The emit key is added as
- -->
- <keys>
- <key k="k1" v="boolean"/>
- <key k="k2" v="string"/>
- <key k="k3" v="int"/>
- <key k="k4" v="long"/>
- </keys>
- <attachmentStrategy>first_only</attachmentStrategy>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-multivalued.xml b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-multivalued.xml
deleted file mode 100644
index eb966b54a..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-multivalued.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.jdbc.JDBCEmitter">
- <name>jdbc</name>
- <connection>CONNECTION_STRING</connection>
- <createTable>create table test
- (path varchar(512) primary key,
- k1 varchar(512));
- </createTable>
- <!-- the jdbc emitter always puts ths emitKey value as the first
- item -->
- <insert>insert into test (path, k1) values (?,?);
- </insert>
- <!-- these are the keys in the metadata object.
- The emitKey is added as the first element in the insert statement.
- Then the these values are added in order.
- They must be in the order of the insert statement.
- -->
- <keys>
- <key k="k1" v="varchar(512)"/>
- </keys>
- <multivaluedFieldStrategy>concatenate</multivaluedFieldStrategy>
- <multivaluedFieldDelimiter>, </multivaluedFieldDelimiter>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-trunc.xml b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-trunc.xml
deleted file mode 100644
index 85eef281c..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-trunc.xml
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.jdbc.JDBCEmitter">
- <name>jdbc</name>
- <connection>CONNECTION_STRING</connection>
- <createTable>create table test
- (path varchar(512) primary key,
- k1 varchar(12));
- </createTable>
- <!-- the jdbc emitter always puts ths emitKey value as the first
- item -->
- <insert>insert into test (path, k1) values (?,?);
- </insert>
- <!-- these are the keys in the metadata object.
- The emitKey is added as the first element in the insert statement.
- Then the these values are added in order.
- They must be in the order of the insert statement.
- -->
- <keys>
- <key k="k1" v="varchar(12)"/>
- </keys>
- <attachmentStrategy>first_only</attachmentStrategy>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter.xml b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter.xml
deleted file mode 100644
index c1a05bdec..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter.xml
+++ /dev/null
@@ -1,54 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.jdbc.JDBCEmitter">
- <name>jdbc</name>
- <connection>CONNECTION_STRING</connection>
- <createTable>create table test
- (path varchar(512) primary key,
- k1 boolean,
- k2 varchar(512),
- k3 integer,
- k4 long,
- k5 bigint,
- k6 timestamp);
- </createTable>
- <!-- the jdbc emitter always puts ths emitKey value as the first
- item -->
- <insert>insert into test (path, k1, k2, k3, k4, k5, k6) values (?,?,?,?,?,?,?);
- </insert>
- <!-- these are the keys in the metadata object.
- The emitKey is added as the first element in the insert statement.
- Then the these values are added in order.
- They must be in the order of the insert statement.
- -->
- <keys>
- <key k="k1" v="boolean"/>
- <key k="k2" v="string"/>
- <key k="k3" v="int"/>
- <key k="k4" v="long"/>
- <key k="k5" v="bigint"/>
- <key k="k6" v="timestamp"/>
- </keys>
- <attachmentStrategy>first_only</attachmentStrategy>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-kafka/pom.xml b/tika-pipes/tika-emitters/tika-emitter-kafka/pom.xml
deleted file mode 100644
index dd7b2d2de..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-kafka/pom.xml
+++ /dev/null
@@ -1,118 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-kafka</artifactId>
- <name>Apache Tika Kafka emitter</name>
-
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${kafka.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.emitter.kafka</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-kafka/src/main/java/org/apache/tika/pipes/emitter/kafka/KafkaEmitter.java b/tika-pipes/tika-emitters/tika-emitter-kafka/src/main/java/org/apache/tika/pipes/emitter/kafka/KafkaEmitter.java
deleted file mode 100644
index cc7f4ea67..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-kafka/src/main/java/org/apache/tika/pipes/emitter/kafka/KafkaEmitter.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.kafka;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerConfig;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-
-/**
- * Emits the now-parsed documents into a specified Apache Kafka topic.
- */
-public class KafkaEmitter extends AbstractEmitter implements Initializable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(KafkaEmitter.class);
-
- private static final ObjectMapper OM = new ObjectMapper();
-
- String topic;
- String bootstrapServers;
-
- String acks = "all";
- int lingerMs = 5000;
- int batchSize = 16384;
- int bufferMemory = 32 * 1024 * 1024;
- String compressionType = "none";
- int connectionsMaxIdleMs = 9 * 60 * 1000;
- int deliveryTimeoutMs = 120 * 1000;
- boolean enableIdempotence = false;
- String interceptorClasses;
- int maxBlockMs = 60 * 1000;
- int maxInFlightRequestsPerConnection = 5;
- int maxRequestSize = 1024 * 1024;
- int metadataMaxAgeMs = 5 * 60 * 1000;
- int requestTimeoutMs = 30 * 1000;
- int retries = Integer.MAX_VALUE;
- int retryBackoffMs = 100;
- int transactionTimeoutMs = 60000;
- String transactionalId;
- String clientId;
- String keySerializer;
- String valueSerializer;
-
- private Producer<String, String> producer;
-
- @Field
- public void setBootstrapServers(String bootstrapServers) {
- this.bootstrapServers = bootstrapServers;
- }
-
- @Field
- public void setAcks(String acks) {
- this.acks = acks;
- }
-
- @Field
- public void setLingerMs(int lingerMs) {
- this.lingerMs = lingerMs;
- }
-
- public void setBatchSize(int batchSize) {
- this.batchSize = batchSize;
- }
-
- @Field
- public void setBufferMemory(int bufferMemory) {
- this.bufferMemory = bufferMemory;
- }
-
- @Field
- public void setClientId(String clientId) {
- this.clientId = clientId;
- }
-
- @Field
- public void setCompressionType(String compressionType) {
- this.compressionType = compressionType;
- }
-
- @Field
- public void setConnectionsMaxIdleMs(int connectionsMaxIdleMs) {
- this.connectionsMaxIdleMs = connectionsMaxIdleMs;
- }
-
- @Field
- public void setDeliveryTimeoutMs(int deliveryTimeoutMs) {
- this.deliveryTimeoutMs = deliveryTimeoutMs;
- }
-
- @Field
- public void setEnableIdempotence(boolean enableIdempotence) {
- this.enableIdempotence = enableIdempotence;
- }
-
- @Field
- public void setInterceptorClasses(String interceptorClasses) {
- this.interceptorClasses = interceptorClasses;
- }
-
- @Field
- public void setMaxBlockMs(int maxBlockMs) {
- this.maxBlockMs = maxBlockMs;
- }
-
- @Field
- public void setMaxInFlightRequestsPerConnection(int maxInFlightRequestsPerConnection) {
- this.maxInFlightRequestsPerConnection = maxInFlightRequestsPerConnection;
- }
-
- @Field
- public void setMaxRequestSize(int maxRequestSize) {
- this.maxRequestSize = maxRequestSize;
- }
-
- @Field
- public void setMetadataMaxAgeMs(int metadataMaxAgeMs) {
- this.metadataMaxAgeMs = metadataMaxAgeMs;
- }
-
- @Field
- public void setRequestTimeoutMs(int requestTimeoutMs) {
- this.requestTimeoutMs = requestTimeoutMs;
- }
-
- @Field
- public void setRetries(int retries) {
- this.retries = retries;
- }
-
- @Field
- public void setRetryBackoffMs(int retryBackoffMs) {
- this.retryBackoffMs = retryBackoffMs;
- }
-
- @Field
- public void setTransactionTimeoutMs(int transactionTimeoutMs) {
- this.transactionTimeoutMs = transactionTimeoutMs;
- }
-
- @Field
- public void setTransactionalId(String transactionalId) {
- this.transactionalId = transactionalId;
- }
-
- @Field
- public void setKeySerializer(String keySerializer) {
- this.keySerializer = keySerializer;
- }
-
- @Field
- public void setValueSerializer(String valueSerializer) {
- this.valueSerializer = valueSerializer;
- }
-
- @Field
- public void setTopic(String topic) {
- this.topic = topic;
- }
-
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext)
- throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.isEmpty()) {
- throw new TikaEmitterException("metadata list must not be null or of size 0");
- }
- for (Metadata metadata : metadataList) {
- LOGGER.debug("about to emit to target topic: ({}) path:({})", topic, emitKey);
-
- Map<String, Object> fields = new HashMap<>();
- for (String n : metadata.names()) {
- String[] vals = metadata.getValues(n);
- if (vals.length > 1) {
- LOGGER.warn("Can only write the first value for key {}. I see {} values.",
- n,
- vals.length);
- }
- fields.put(n, vals[0]);
- }
-
- producer.send(new ProducerRecord<>(topic, emitKey, OM.writeValueAsString(fields)));
- }
- }
-
- private void safePut(Properties props, String key, Object val) {
- if (val != null) {
- props.put(key, val);
- }
- }
-
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
-
- // create instance for properties to access producer configs
- Properties props = new Properties();
-
- //Assign localhost id
- safePut(props, ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
-
- //Set acknowledgements for producer requests.
- safePut(props, ProducerConfig.ACKS_CONFIG, acks);
-
- //If the request fails, the producer can automatically retry,
- safePut(props, ProducerConfig.RETRIES_CONFIG, retries);
-
- //Specify buffer size in config
- safePut(props, ProducerConfig.BATCH_SIZE_CONFIG, batchSize);
-
- //Reduce the no of requests less than 0
- safePut(props, ProducerConfig.LINGER_MS_CONFIG, lingerMs);
-
- //The buffer.memory controls the total amount of memory available to the producer for buffering.
- safePut(props, ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory);
-
- safePut(props, ProducerConfig.CLIENT_ID_CONFIG, clientId);
- safePut(props, ProducerConfig.COMPRESSION_TYPE_CONFIG, compressionType);
- safePut(props, ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, deliveryTimeoutMs);
- safePut(props, ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, enableIdempotence);
- safePut(props, ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, interceptorClasses);
- safePut(props, ProducerConfig.MAX_BLOCK_MS_CONFIG, maxBlockMs);
- safePut(props, ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, maxInFlightRequestsPerConnection);
- safePut(props, ProducerConfig.MAX_REQUEST_SIZE_CONFIG, maxRequestSize);
- safePut(props, ProducerConfig.METADATA_MAX_AGE_CONFIG, metadataMaxAgeMs);
- safePut(props, ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, requestTimeoutMs);
- safePut(props, ProducerConfig.RETRY_BACKOFF_MS_CONFIG, retryBackoffMs);
- safePut(props, ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, transactionTimeoutMs);
- safePut(props, ProducerConfig.TRANSACTIONAL_ID_CONFIG, transactionalId);
-
- safePut(props, ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
- serializerClass(keySerializer, StringSerializer.class));
- safePut(props, ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
- serializerClass(valueSerializer, StringSerializer.class));
-
- producer = new KafkaProducer<>(props);
- }
-
- private Object serializerClass(String className, Class defaultClass) {
- try {
- return className == null ? defaultClass : Class.forName(className);
- } catch (ClassNotFoundException e) {
- LOGGER.error("Could not find key serializer class: {}", className);
- return null;
- }
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler)
- throws TikaConfigException {
- mustNotBeEmpty("topic", this.topic);
- mustNotBeEmpty("server", this.bootstrapServers);
- }
-
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/pom.xml b/tika-pipes/tika-emitters/tika-emitter-opensearch/pom.xml
deleted file mode 100644
index 7853eceb9..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/pom.xml
+++ /dev/null
@@ -1,135 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-opensearch</artifactId>
- <name>Apache Tika OpenSearch emitter</name>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-httpclient-commons</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-serialization</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-io</artifactId>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-http</artifactId>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.emitter.opensearch</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/JsonResponse.java b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/JsonResponse.java
deleted file mode 100644
index 9a5d36407..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/JsonResponse.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.opensearch;
-
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-public class JsonResponse {
-
- private final int status;
- private final String msg;
- private final JsonNode root;
-
- public JsonResponse(int status, JsonNode root) {
- this.status = status;
- this.root = root;
- this.msg = null;
- }
-
- public JsonResponse(int status, String msg) {
- this.status = status;
- this.msg = msg;
- this.root = null;
- }
-
- public int getStatus() {
- return status;
- }
-
- public String getMsg() {
- return msg;
- }
-
- public JsonNode getJson() {
- return root;
- }
-
- @Override
- public String toString() {
- return "JsonResponse{" +
- "status=" + status +
- ", msg='" + msg + '\'' +
- ", root=" + root +
- '}';
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClient.java b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClient.java
deleted file mode 100644
index d26ac46e0..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClient.java
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.opensearch;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.io.StringWriter;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.UUID;
-
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.client.TikaClientException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.pipes.core.emitter.EmitData;
-import org.apache.tika.utils.StringUtils;
-
-public class OpenSearchClient {
-
- private static final Logger LOG = LoggerFactory.getLogger(OpenSearchEmitter.class);
-
- //this includes the full url and the index, should not end in /
- //e.g. https://localhost:9200/my-index
- protected final String openSearchUrl;
- protected final HttpClient httpClient;
- private final OpenSearchEmitter.AttachmentStrategy attachmentStrategy;
-
- private final MetadataToJsonWriter metadataToJsonWriter;
- private final String embeddedFileFieldName;
- protected OpenSearchClient(String openSearchUrl, HttpClient httpClient,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy,
- OpenSearchEmitter.UpdateStrategy updateStrategy,
- String embeddedFileFieldName) {
- this.openSearchUrl = openSearchUrl;
- this.httpClient = httpClient;
- this.attachmentStrategy = attachmentStrategy;
- this.metadataToJsonWriter = (updateStrategy == OpenSearchEmitter.UpdateStrategy.OVERWRITE) ?
- new InsertMetadataToJsonWriter() : new UpsertMetadataToJsonWriter();
- this.embeddedFileFieldName = embeddedFileFieldName;
- }
-
-
- public void emitDocuments(List<? extends EmitData> emitData) throws IOException, TikaClientException {
- StringBuilder json = new StringBuilder();
- for (EmitData d : emitData) {
- appendDoc(d.getEmitKey().getEmitKey(), d.getMetadataList(), json);
- }
- emitJson(json);
- }
-
- private void emitJson(StringBuilder json) throws IOException, TikaClientException {
- String requestUrl = openSearchUrl + "/_bulk";
- JsonResponse response = postJson(requestUrl, json.toString());
- if (response.getStatus() != 200) {
- throw new TikaClientException(response.getMsg());
- } else {
- //if there's a single error, throw the full json.
- //this has not been thoroughly tested with versions of es < 7
- JsonNode errorNode = response.getJson().get("errors");
- if (errorNode.asText().equals("true")) {
- throw new TikaClientException(response.getJson().toString());
- }
- }
- }
-
-
- public void emitDocument(String emitKey, List<Metadata> metadataList) throws IOException,
- TikaClientException {
-
- StringBuilder json = new StringBuilder();
- appendDoc(emitKey, metadataList, json);
- emitJson(json);
- }
-
- private void appendDoc(String emitKey, List<Metadata> metadataList, StringBuilder json)
- throws IOException {
- int i = 0;
- String routing = (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.PARENT_CHILD) ?
- emitKey : null;
-
- for (Metadata metadata : metadataList) {
- StringBuilder id = new StringBuilder(emitKey);
- if (i > 0) {
- id.append("-").append(UUID.randomUUID());
- }
- String indexJson = metadataToJsonWriter.getBulkJson(id.toString(), routing);
- json.append(indexJson).append("\n");
- if (i == 0) {
- json.append(metadataToJsonWriter.writeContainer(metadata, attachmentStrategy));
- } else {
- json.append(metadataToJsonWriter.writeEmbedded(metadata, attachmentStrategy, emitKey,
- embeddedFileFieldName));
- }
- json.append("\n");
- i++;
- }
- }
-
- //Only here for testing. These may disappear without notice in the future.
- protected static String metadataToJsonContainerInsert(Metadata metadata,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy)
- throws IOException {
- return new InsertMetadataToJsonWriter().writeContainer(metadata, attachmentStrategy);
- }
-
- //Only here for testing. These may disappear without notice in the future.
- protected static String metadataToJsonEmbeddedInsert(Metadata metadata,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy,
- String emitKey, String embeddedFileFieldName)
- throws IOException {
- return new InsertMetadataToJsonWriter().writeEmbedded(metadata,
- attachmentStrategy, emitKey, embeddedFileFieldName);
- }
-
- public JsonResponse postJson(String url, String json) throws IOException {
- HttpPost httpRequest = new HttpPost(url);
- StringEntity entity = new StringEntity(json, StandardCharsets.UTF_8);
- httpRequest.setEntity(entity);
- httpRequest.setHeader("Accept", "application/json");
- httpRequest.setHeader("Content-type", "application/json; charset=utf-8");
- //At one point, this was required because of connection already
- // bound exceptions on windows :(
- //httpPost.setHeader("Connection", "close");
-
- //try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
-
- HttpResponse response = null;
- try {
- response = httpClient.execute(httpRequest);
- int status = response.getStatusLine().getStatusCode();
- if (status == 200) {
- try (Reader reader = new BufferedReader(
- new InputStreamReader(response.getEntity().getContent(),
- StandardCharsets.UTF_8))) {
- ObjectMapper mapper = new ObjectMapper();
- JsonNode node = mapper.readTree(reader);
- if (LOG.isTraceEnabled()) {
- LOG.trace("node: {}", node);
- }
- return new JsonResponse(200, node);
- }
- } else {
- return new JsonResponse(status,
- new String(EntityUtils.toByteArray(response.getEntity()),
- StandardCharsets.UTF_8));
- }
- } finally {
- if (response != null && response instanceof CloseableHttpResponse) {
- ((CloseableHttpResponse)response).close();
- }
- httpRequest.releaseConnection();
- }
- }
-
- private interface MetadataToJsonWriter {
- String writeContainer(Metadata metadata, OpenSearchEmitter.AttachmentStrategy attachmentStrategy)
- throws IOException;
-
- String writeEmbedded(Metadata metadata, OpenSearchEmitter.AttachmentStrategy attachmentStrategy,
- String emitKey, String embeddedFileFieldName) throws IOException;
-
- String getBulkJson(String id, String routing) throws IOException;
- }
-
- private static class InsertMetadataToJsonWriter implements MetadataToJsonWriter {
-
- @Override
- public String writeContainer(Metadata metadata,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy)
- throws IOException {
- StringWriter writer = new StringWriter();
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
- jsonGenerator.writeStartObject();
- writeMetadata(metadata, jsonGenerator);
- if (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.PARENT_CHILD) {
- jsonGenerator.writeStringField("relation_type", "container");
- }
- jsonGenerator.writeEndObject();
- }
- return writer.toString();
- }
-
- @Override
- public String writeEmbedded(Metadata metadata,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy,
- String emitKey, String embeddedFileFieldName)
- throws IOException {
- StringWriter writer = new StringWriter();
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
- jsonGenerator.writeStartObject();
-
- writeMetadata(metadata, jsonGenerator);
- if (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.PARENT_CHILD) {
- jsonGenerator.writeObjectFieldStart("relation_type");
- jsonGenerator.writeStringField("name", embeddedFileFieldName);
- jsonGenerator.writeStringField("parent", emitKey);
- //end the relation type object
- jsonGenerator.writeEndObject();
- } else if (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.SEPARATE_DOCUMENTS) {
- jsonGenerator.writeStringField("parent", emitKey);
- }
- //end the metadata object
- jsonGenerator.writeEndObject();
- }
- return writer.toString();
- }
-
- @Override
- public String getBulkJson(String id, String routing) throws IOException {
- StringWriter writer = new StringWriter();
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
- jsonGenerator.writeStartObject();
- jsonGenerator.writeObjectFieldStart("index");
- jsonGenerator.writeStringField("_id", id);
- if (!StringUtils.isEmpty(routing)) {
- jsonGenerator.writeStringField("routing", routing);
- }
- jsonGenerator.writeEndObject();
- jsonGenerator.writeEndObject();
- }
- return writer.toString();
- }
- }
-
- private static class UpsertMetadataToJsonWriter implements MetadataToJsonWriter {
-
- @Override
- public String writeContainer(Metadata metadata,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy)
- throws IOException {
- StringWriter writer = new StringWriter();
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
- jsonGenerator.writeStartObject();
- jsonGenerator.writeObjectFieldStart("doc");
- writeMetadata(metadata, jsonGenerator);
- if (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.PARENT_CHILD) {
- jsonGenerator.writeStringField("relation_type", "container");
- }
- jsonGenerator.writeEndObject();
- jsonGenerator.writeBooleanField("doc_as_upsert", true);
- jsonGenerator.writeEndObject();
- }
- return writer.toString();
- }
-
- @Override
- public String writeEmbedded(Metadata metadata,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy,
- String emitKey, String embeddedFileFieldName)
- throws IOException {
- StringWriter writer = new StringWriter();
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
- jsonGenerator.writeStartObject();
- jsonGenerator.writeObjectFieldStart("doc");
- writeMetadata(metadata, jsonGenerator);
- if (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.PARENT_CHILD) {
- jsonGenerator.writeObjectFieldStart("relation_type");
- jsonGenerator.writeStringField("name", embeddedFileFieldName);
- jsonGenerator.writeStringField("parent", emitKey);
- //end the relation type object
- jsonGenerator.writeEndObject();
- } else if (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.SEPARATE_DOCUMENTS) {
- jsonGenerator.writeStringField("parent", emitKey);
- }
- //end the "doc"
- jsonGenerator.writeEndObject();
- jsonGenerator.writeBooleanField("doc_as_upsert", true);
- //end the metadata object
- jsonGenerator.writeEndObject();
- }
- return writer.toString();
- }
-
- @Override
- public String getBulkJson(String id, String routing) throws IOException {
- StringWriter writer = new StringWriter();
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
- jsonGenerator.writeStartObject();
- jsonGenerator.writeObjectFieldStart("update");
- jsonGenerator.writeStringField("_id", id);
- if (!StringUtils.isEmpty(routing)) {
- jsonGenerator.writeStringField("routing", routing);
- }
- jsonGenerator.writeNumberField("retry_on_conflict", 3);
- jsonGenerator.writeEndObject();
- jsonGenerator.writeEndObject();
- }
- return writer.toString();
- }
- }
-
- private static void writeMetadata(Metadata metadata, JsonGenerator jsonGenerator) throws IOException {
- //writes the metadata without the start { or the end }
- //to allow for other fields to be added
- for (String n : metadata.names()) {
- String[] vals = metadata.getValues(n);
- if (vals.length == 1) {
- jsonGenerator.writeStringField(n, vals[0]);
- } else {
- jsonGenerator.writeArrayFieldStart(n);
- for (String v : vals) {
- jsonGenerator.writeString(v);
- }
- jsonGenerator.writeEndArray();
- }
- }
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchEmitter.java b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchEmitter.java
deleted file mode 100644
index 3b8c21b8d..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchEmitter.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.opensearch;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.client.HttpClientFactory;
-import org.apache.tika.client.TikaClientException;
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.EmitData;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-import org.apache.tika.utils.StringUtils;
-
-
-public class OpenSearchEmitter extends AbstractEmitter implements Initializable {
-
-
- public enum AttachmentStrategy {
- SEPARATE_DOCUMENTS, PARENT_CHILD,
- //anything else?
- }
-
- public enum UpdateStrategy {
- OVERWRITE, UPSERT
- //others?
- }
-
- public static String DEFAULT_EMBEDDED_FILE_FIELD_NAME = "embedded";
- private static final Logger LOG = LoggerFactory.getLogger(OpenSearchEmitter.class);
- private AttachmentStrategy attachmentStrategy = AttachmentStrategy.PARENT_CHILD;
-
- private UpdateStrategy updateStrategy = UpdateStrategy.OVERWRITE;
- private String openSearchUrl = null;
- private String idField = "_id";
- private int commitWithin = 1000;
- private OpenSearchClient openSearchClient;
- private final HttpClientFactory httpClientFactory;
- private String embeddedFileFieldName = DEFAULT_EMBEDDED_FILE_FIELD_NAME;
-
- public OpenSearchEmitter() throws TikaConfigException {
- httpClientFactory = new HttpClientFactory();
- }
-
- @Override
- public void emit(List<? extends EmitData> emitData) throws IOException, TikaEmitterException {
- if (emitData == null || emitData.size() == 0) {
- LOG.debug("metadataList is null or empty");
- return;
- }
- try {
- LOG.debug("about to emit {} docs", emitData.size());
- openSearchClient.emitDocuments(emitData);
- LOG.info("successfully emitted {} docs", emitData.size());
- } catch (TikaClientException e) {
- LOG.warn("problem emitting docs", e);
- throw new TikaEmitterException(e.getMessage());
- }
- }
-
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext)
- throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.size() == 0) {
- LOG.debug("metadataList is null or empty");
- return;
- }
- try {
- LOG.debug("about to emit one doc");
- openSearchClient.emitDocument(emitKey, metadataList);
- LOG.info("successfully emitted one doc");
- } catch (TikaClientException e) {
- LOG.warn("problem emitting doc", e);
- throw new TikaEmitterException("failed to add document", e);
- }
- }
-
-
- /**
- * Options: SEPARATE_DOCUMENTS, PARENT_CHILD. Default is "SEPARATE_DOCUMENTS".
- * All embedded documents are treated as independent documents.
- * PARENT_CHILD requires a schema to be set up for the relationship type;
- * all embedded objects (no matter how deeply nested) will have a single
- * parent of the main container document.
- *
- * If you want to concatenate the content of embedded files and ignore
- * the metadata of embedded files, set
- * {@link org.apache.tika.pipes.HandlerConfig}'s parseMode to
- * {@link org.apache.tika.pipes.HandlerConfig.PARSE_MODE#CONCATENATE}
- * in your {@link org.apache.tika.pipes.FetchEmitTuple} or in the
- * <parseMode> element in your {@link org.apache.tika.pipes.pipesiterator.PipesIterator}
- * configuration.
- */
- @Field
- public void setAttachmentStrategy(String attachmentStrategy) {
- this.attachmentStrategy = AttachmentStrategy.valueOf(attachmentStrategy);
- }
-
-
- @Field
- public void setConnectionTimeout(int connectionTimeout) {
- httpClientFactory.setConnectTimeout(connectionTimeout);
- }
-
- @Field
- public void setSocketTimeout(int socketTimeout) {
- httpClientFactory.setSocketTimeout(socketTimeout);
- }
-
- public int getCommitWithin() {
- return commitWithin;
- }
-
- @Field
- public void setCommitWithin(int commitWithin) {
- this.commitWithin = commitWithin;
- }
-
- /**
- * Specify the field in the first Metadata that should be
- * used as the id field for the document.
- *
- * @param idField
- */
- @Field
- public void setIdField(String idField) {
- this.idField = idField;
- }
-
- //this is the full url, including the collection, e.g. https://localhost:9200/my-collection
- @Field
- public void setOpenSearchUrl(String openSearchUrl) {
- this.openSearchUrl = openSearchUrl;
- }
-
- //TODO -- add other httpclient configurations??
- @Field
- public void setUserName(String userName) {
- httpClientFactory.setUserName(userName);
- }
-
- @Field
- public void setPassword(String password) {
- httpClientFactory.setPassword(password);
- }
-
- @Field
- public void setAuthScheme(String authScheme) {
- httpClientFactory.setAuthScheme(authScheme);
- }
-
- @Field
- public void setProxyHost(String proxyHost) {
- httpClientFactory.setProxyHost(proxyHost);
- }
-
- @Field
- public void setProxyPort(int proxyPort) {
- httpClientFactory.setProxyPort(proxyPort);
- }
-
- public void setUpdateStrategy(UpdateStrategy updateStrategy) {
- this.updateStrategy = updateStrategy;
- }
-
- public void setUpdateStrategy(String strategy) throws TikaConfigException {
- switch (strategy.toLowerCase(Locale.US)) {
- case "overwrite" :
- setUpdateStrategy(UpdateStrategy.OVERWRITE);
- break;
- case "upsert" :
- setUpdateStrategy(UpdateStrategy.UPSERT);
- break;
- default :
- throw new TikaConfigException("'overwrite' and 'upsert' are the two options so " +
- "far. I regret I don't understand: " + strategy);
- }
- }
-
- /**
- * If using the {@link AttachmentStrategy#PARENT_CHILD}, this is the field name
- * used to store the child documents. Note that we artificially flatten all embedded
- * documents, no matter how nested in the container document, into direct children
- * of the root document.
- *
- * @param embeddedFileFieldName
- */
- @Field
- public void setEmbeddedFileFieldName(String embeddedFileFieldName) {
- this.embeddedFileFieldName = embeddedFileFieldName;
- }
-
-
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- if (StringUtils.isBlank(openSearchUrl)) {
- throw new TikaConfigException("Must specify an open search url!");
- } else {
- openSearchClient =
- new OpenSearchClient(openSearchUrl,
- httpClientFactory.build(), attachmentStrategy, updateStrategy,
- embeddedFileFieldName);
- }
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler)
- throws TikaConfigException {
- mustNotBeEmpty("openSearchUrl", this.openSearchUrl);
- mustNotBeEmpty("idField", this.idField);
- }
-
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClientTest.java b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClientTest.java
deleted file mode 100644
index 994ff0044..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClientTest.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.opensearch;
-
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.TikaTest;
-import org.apache.tika.metadata.Metadata;
-
-public class OpenSearchClientTest extends TikaTest {
-
- @Test
- public void testSerialization() throws Exception {
- Metadata metadata = new Metadata();
- metadata.add("authors", "author1");
- metadata.add("authors", "author2");
- metadata.add("title", "title1");
- for (OpenSearchEmitter.AttachmentStrategy strategy :
- OpenSearchEmitter.AttachmentStrategy.values()) {
- String json = OpenSearchClient.metadataToJsonContainerInsert(metadata,
- strategy);
- assertContains("author1", json);
- assertContains("author2", json);
- assertContains("authors", json);
- assertContains("title1", json);
- }
- for (OpenSearchEmitter.AttachmentStrategy strategy :
- OpenSearchEmitter.AttachmentStrategy.values()) {
- String json = OpenSearchClient.metadataToJsonEmbeddedInsert(metadata, strategy,
- "myEmitKey", OpenSearchEmitter.DEFAULT_EMBEDDED_FILE_FIELD_NAME);
- assertContains("author1", json);
- assertContains("author2", json);
- assertContains("authors", json);
- assertContains("title1", json);
- }
-
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/resources/tika-config-simple-emitter.xml b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/resources/tika-config-simple-emitter.xml
deleted file mode 100644
index f6530a9e9..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/resources/tika-config-simple-emitter.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <metadataFilters>
- <metadataFilter class="org.apache.tika.metadata.filter.FieldNameMappingFilter">
- <mappings>
- <mapping from="X-TIKA:content" to="content"/>
- <mapping from="X-TIKA:embedded_resource_path" to="embedded_path"/>
- <mapping from="Content-Length" to="length"/>
- <mapping from="dc:creator" to="creators"/>
- <mapping from="dc:title" to="title"/>
- </mappings>
- </metadataFilter>
- </metadataFilters>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.opensearch.OpenSearchEmitter">
- <name>opensearch1</name>
- <url>http://localhost:9200/tika-test</url>
- <attachmentStrategy>concatenate-content</attachmentStrategy>
- <contentField>content</contentField>
- <commitWithin>10</commitWithin>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml b/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml
deleted file mode 100644
index b48a8c302..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml
+++ /dev/null
@@ -1,112 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-s3</artifactId>
- <name>Apache Tika S3 emitter</name>
-
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.emitter.s3</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java b/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
deleted file mode 100644
index c454c6c6a..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.s3;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-import java.util.List;
-import java.util.Map;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.InstanceProfileCredentialsProvider;
-import com.amazonaws.auth.profile.ProfileCredentialsProvider;
-import com.amazonaws.client.builder.AwsClientBuilder;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TemporaryResources;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.StreamEmitter;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-import org.apache.tika.serialization.JsonMetadataList;
-import org.apache.tika.utils.StringUtils;
-
-/**
- * Emits to existing s3 bucket
- * <pre class="prettyprint">
- * <properties>
- * <emitters>
- * <emitter class="org.apache.tika.pipes.emitter.s3.S3Emitter>
- * <params>
- * <!-- required -->
- * <param name="name" type="string">s3e</param>
- * <!-- required -->
- * <param name="region" type="string">us-east-1</param>
- * <!-- required -->
- * <param name="credentialsProvider"
- * type="string">(profile|instance)</param>
- * <!-- required if credentialsProvider=profile-->
- * <param name="profile" type="string">my-profile</param>
- * <!-- required -->
- * <param name="bucket" type="string">my-bucket</param>
- * <!-- optional; prefix to add to the path before emitting;
- * default is no prefix -->
- * <param name="prefix" type="string">my-prefix</param>
- * <!-- optional; default is 'json' this will be added to the SOURCE_PATH
- * if no emitter key is specified. Do not add a "."
- * before the extension -->
- * <param name="fileExtension" type="string">json</param>
- * <!-- optional; default is 'true'-- whether to copy the
- * json to a local file before putting to s3 -->
- * <param name="spoolToTemp" type="bool">true</param>
- * </params>
- * </emitter>
- * </emitters>
- * </properties></pre>
- */
-public class S3Emitter extends AbstractEmitter implements Initializable, StreamEmitter {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(S3Emitter.class);
- private String region;
- private String profile;
- private String bucket;
- private String credentialsProvider;
- private String accessKey;
- private String secretKey;
- private String endpointConfigurationService;
- private String fileExtension = "json";
- private boolean spoolToTemp = true;
- private String prefix = null;
- private int maxConnections = ClientConfiguration.DEFAULT_MAX_CONNECTIONS;
- private boolean pathStyleAccessEnabled = false;
- private AmazonS3 s3Client;
-
- /**
- * Requires the src-bucket/path/to/my/file.txt in the {@link TikaCoreProperties#SOURCE_PATH}.
- *
- * @param metadataList
- * @throws IOException
- * @throws TikaException
- */
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext) throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.size() == 0) {
- throw new TikaEmitterException("metadata list must not be null or of size 0");
- }
-
- if (!spoolToTemp) {
- UnsynchronizedByteArrayOutputStream bos = UnsynchronizedByteArrayOutputStream
- .builder()
- .get();
- try (Writer writer = new BufferedWriter(new OutputStreamWriter(bos, StandardCharsets.UTF_8))) {
- JsonMetadataList.toJson(metadataList, writer);
- } catch (IOException e) {
- throw new TikaEmitterException("can't jsonify", e);
- }
- byte[] bytes = bos.toByteArray();
- try (InputStream is = TikaInputStream.get(bytes)) {
- emit(emitKey, is, new Metadata(), parseContext);
- }
- } else {
- try (TemporaryResources tmp = new TemporaryResources()) {
- Path tmpPath = tmp.createTempFile();
- try (Writer writer = Files.newBufferedWriter(tmpPath, StandardCharsets.UTF_8, StandardOpenOption.CREATE)) {
- JsonMetadataList.toJson(metadataList, writer);
- } catch (IOException e) {
- throw new TikaEmitterException("can't jsonify", e);
- }
- try (InputStream is = TikaInputStream.get(tmpPath)) {
- emit(emitKey, is, new Metadata(), parseContext);
- }
- }
- }
- }
-
- /**
- * @param path -- object path, not including the bucket
- * @param is inputStream to copy
- * @param userMetadata this will be written to the s3 ObjectMetadata's userMetadata
- * @throws TikaEmitterException or IOexception if there is a Runtime s3 client exception
- */
- @Override
- public void emit(String path, InputStream is, Metadata userMetadata, ParseContext parseContext) throws IOException, TikaEmitterException {
-
- if (!StringUtils.isBlank(prefix)) {
- path = prefix + "/" + path;
- }
-
- if (!StringUtils.isBlank(fileExtension)) {
- path += "." + fileExtension;
- }
-
- LOGGER.debug("about to emit to target bucket: ({}) path:({})", bucket, path);
-
- ObjectMetadata objectMetadata = new ObjectMetadata();
- for (String n : userMetadata.names()) {
- String[] vals = userMetadata.getValues(n);
- if (vals.length > 1) {
- LOGGER.warn("Can only write the first value for key {}. I see {} values.", n, vals.length);
- }
- objectMetadata.addUserMetadata(n, vals[0]);
- }
- //In practice, sending a file is more robust
- //We ran into stream reset issues during digesting, and aws doesn't
- //like putObjects for streams without lengths
- if (is instanceof TikaInputStream) {
- if (((TikaInputStream) is).hasFile()) {
- try {
- PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, path, ((TikaInputStream) is).getFile()).withMetadata(objectMetadata);
- s3Client.putObject(putObjectRequest);
- } catch (IOException e) {
- throw new TikaEmitterException("exception sending underlying file", e);
- }
- return;
- }
- }
- try {
- s3Client.putObject(bucket, path, is, objectMetadata);
- } catch (AmazonClientException e) {
- throw new IOException("problem writing s3object", e);
- }
- }
-
- /**
- * Whether or not to spool the metadatalist to a tmp file before putting object.
- * Default: <code>true</code>. If this is set to <code>false</code>,
- * this emitter writes the json object to memory and then puts that into s3.
- *
- * @param spoolToTemp
- */
- @Field
- public void setSpoolToTemp(boolean spoolToTemp) {
- this.spoolToTemp = spoolToTemp;
- }
-
- @Field
- public void setRegion(String region) {
- this.region = region;
- }
-
- @Field
- public void setProfile(String profile) {
- this.profile = profile;
- }
-
- @Field
- public void setBucket(String bucket) {
- this.bucket = bucket;
- }
-
- @Field
- public void setPrefix(String prefix) {
- //strip final "/" if it exists
- if (prefix.endsWith("/")) {
- this.prefix = prefix.substring(0, prefix.length() - 1);
- } else {
- this.prefix = prefix;
- }
- }
-
- @Field
- public void setCredentialsProvider(String credentialsProvider) {
- if (!credentialsProvider.equals("profile") && !credentialsProvider.equals("instance") && !credentialsProvider.equals("key_secret")) {
- throw new IllegalArgumentException("credentialsProvider must be either 'profile', 'instance' or 'key_secret'");
- }
- this.credentialsProvider = credentialsProvider;
- }
-
- /**
- * If you want to customize the output file's file extension.
- * Do not include the "."
- *
- * @param fileExtension
- */
- @Field
- public void setFileExtension(String fileExtension) {
- this.fileExtension = fileExtension;
- }
-
- @Field
- public void setAccessKey(String accessKey) {
- this.accessKey = accessKey;
- }
-
- @Field
- public void setSecretKey(String secretKey) {
- this.secretKey = secretKey;
- }
-
- /**
- * maximum number of http connections allowed. This should be
- * greater than or equal to the number of threads emitting to S3.
- *
- * @param maxConnections
- */
- @Field
- public void setMaxConnections(int maxConnections) {
- this.maxConnections = maxConnections;
- }
-
- @Field
- public void setEndpointConfigurationService(String endpointConfigurationService) {
- this.endpointConfigurationService = endpointConfigurationService;
- }
-
- /**
- * This initializes the s3 client. Note, we wrap S3's RuntimeExceptions,
- * e.g. AmazonClientException in a TikaConfigException.
- *
- * @param params params to use for initialization
- * @throws TikaConfigException
- */
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- //params have already been set...ignore them
- AWSCredentialsProvider provider;
- if ("instance".equals(credentialsProvider)) {
- provider = InstanceProfileCredentialsProvider.getInstance();
- } else if ("profile".equals(credentialsProvider)) {
- provider = new ProfileCredentialsProvider(profile);
- } else if (credentialsProvider.equals("key_secret")) {
- provider = new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey));
- } else {
- throw new TikaConfigException("credentialsProvider must be set and " + "must be either 'instance', 'profile' or 'key_secret'");
- }
- ClientConfiguration clientConfig = new ClientConfiguration().withMaxConnections(maxConnections);
- try {
- AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder
- .standard()
- .withClientConfiguration(clientConfig)
- .withCredentials(provider)
- .withPathStyleAccessEnabled(pathStyleAccessEnabled);
- if (!StringUtils.isBlank(endpointConfigurationService)) {
- amazonS3ClientBuilder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpointConfigurationService, region));
- } else {
- amazonS3ClientBuilder.withRegion(region);
- }
- s3Client = amazonS3ClientBuilder.build();
- } catch (AmazonClientException e) {
- throw new TikaConfigException("can't initialize s3 emitter", e);
- }
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
- mustNotBeEmpty("bucket", this.bucket);
- mustNotBeEmpty("region", this.region);
- }
-
- @Field
- public void setPathStyleAccessEnabled(boolean pathStyleAccessEnabled) {
- this.pathStyleAccessEnabled = pathStyleAccessEnabled;
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml b/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
deleted file mode 100644
index b74dd0a67..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
+++ /dev/null
@@ -1,117 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-solr</artifactId>
- <name>Apache Tika Apache Solr emitter</name>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-httpclient-commons</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-solrj</artifactId>
- <version>${solrj.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.emitter.solr</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java
deleted file mode 100644
index 7f46304d5..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.solr;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.Http2SolrClient;
-import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.response.UpdateResponse;
-import org.apache.solr.common.SolrInputDocument;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.client.HttpClientFactory;
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.EmitData;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-import org.apache.tika.utils.StringUtils;
-
-
-public class SolrEmitter extends AbstractEmitter implements Initializable {
-
- public static String DEFAULT_EMBEDDED_FILE_FIELD_NAME = "embedded";
- private static final Logger LOG = LoggerFactory.getLogger(SolrEmitter.class);
- private final HttpClientFactory httpClientFactory;
- private AttachmentStrategy attachmentStrategy = AttachmentStrategy.PARENT_CHILD;
- private UpdateStrategy updateStrategy = UpdateStrategy.ADD;
- private String solrCollection;
- /**
- * You can specify solrUrls, or you can specify solrZkHosts and use use zookeeper to determine the solr server urls.
- */
- private List<String> solrUrls;
- private List<String> solrZkHosts;
- private String solrZkChroot;
- private String idField = "id";
- private int commitWithin = 1000;
- private int connectionTimeout = 10000;
- private int socketTimeout = 60000;
- private SolrClient solrClient;
- private String embeddedFileFieldName = DEFAULT_EMBEDDED_FILE_FIELD_NAME;
-
- public SolrEmitter() throws TikaConfigException {
- httpClientFactory = new HttpClientFactory();
- }
-
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext)
- throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.size() == 0) {
- LOG.warn("metadataList is null or empty");
- return;
- }
- List<SolrInputDocument> docsToUpdate = new ArrayList<>();
- addMetadataAsSolrInputDocuments(emitKey, metadataList, docsToUpdate);
- emitSolrBatch(docsToUpdate);
- }
-
- private void addMetadataAsSolrInputDocuments(String emitKey, List<Metadata> metadataList,
- List<SolrInputDocument> docsToUpdate)
- throws IOException, TikaEmitterException {
- SolrInputDocument solrInputDocument = new SolrInputDocument();
- solrInputDocument.setField(idField, emitKey);
- if (updateStrategy == UpdateStrategy.UPDATE_MUST_EXIST) {
- solrInputDocument.setField("_version_", 1);
- } else if (updateStrategy == UpdateStrategy.UPDATE_MUST_NOT_EXIST) {
- solrInputDocument.setField("_version_", -1);
- }
- if (metadataList.size() == 1) {
- addMetadataToSolrInputDocument(metadataList.get(0), solrInputDocument, updateStrategy);
- docsToUpdate.add(solrInputDocument);
- } else if (attachmentStrategy == AttachmentStrategy.PARENT_CHILD) {
- addMetadataToSolrInputDocument(metadataList.get(0), solrInputDocument, updateStrategy);
- List<SolrInputDocument> children = new ArrayList<>();
- for (int i = 1; i < metadataList.size(); i++) {
- SolrInputDocument childSolrInputDocument = new SolrInputDocument();
- Metadata m = metadataList.get(i);
- childSolrInputDocument
- .setField(idField, emitKey + "-" + UUID.randomUUID().toString());
- addMetadataToSolrInputDocument(m, childSolrInputDocument, updateStrategy);
- children.add(childSolrInputDocument);
- }
- solrInputDocument.setField(embeddedFileFieldName, children);
- docsToUpdate.add(solrInputDocument);
- } else if (attachmentStrategy == AttachmentStrategy.SEPARATE_DOCUMENTS) {
- addMetadataToSolrInputDocument(metadataList.get(0), solrInputDocument, updateStrategy);
- docsToUpdate.add(solrInputDocument);
- for (int i = 1; i < metadataList.size(); i++) {
- SolrInputDocument childSolrInputDocument = new SolrInputDocument();
- Metadata m = metadataList.get(i);
- childSolrInputDocument.setField(idField,
- solrInputDocument.get(idField).getValue() + "-" + UUID.randomUUID().toString());
- addMetadataToSolrInputDocument(m, childSolrInputDocument, updateStrategy);
- docsToUpdate.add(childSolrInputDocument);
- }
- } else {
- throw new IllegalArgumentException(
- "I don't yet support this attachment strategy: " + attachmentStrategy);
- }
- }
-
- @Override
- public void emit(List<? extends EmitData> batch) throws IOException, TikaEmitterException {
- if (batch == null || batch.size() == 0) {
- LOG.warn("batch is null or empty");
- return;
- }
- List<SolrInputDocument> docsToUpdate = new ArrayList<>();
- for (EmitData d : batch) {
- addMetadataAsSolrInputDocuments(d.getEmitKey().getEmitKey(), d.getMetadataList(),
- docsToUpdate);
- }
- emitSolrBatch(docsToUpdate);
- }
-
- private void emitSolrBatch(List<SolrInputDocument> docsToUpdate)
- throws IOException, TikaEmitterException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Emitting solr doc batch: {}", docsToUpdate);
- }
- if (!docsToUpdate.isEmpty()) {
- try {
- UpdateRequest req = new UpdateRequest();
- req.add(docsToUpdate);
- req.setCommitWithin(commitWithin);
- req.setParam("failOnVersionConflicts", "false");
- UpdateResponse updateResponse = req.process(solrClient, solrCollection);
- LOG.debug("update response: " + updateResponse);
- if (updateResponse.getStatus() != 0) {
- throw new TikaEmitterException("Bad status: " + updateResponse);
- }
- } catch (Exception e) {
- throw new TikaEmitterException("Could not add batch to solr", e);
- }
- }
- }
-
- private void addMetadataToSolrInputDocument(Metadata metadata,
- SolrInputDocument solrInputDocument,
- UpdateStrategy updateStrategy) {
- for (String n : metadata.names()) {
- String[] vals = metadata.getValues(n);
- if (vals.length == 0) {
- continue;
- } else if (vals.length == 1) {
- if (updateStrategy == UpdateStrategy.ADD) {
- solrInputDocument.setField(n, vals[0]);
- } else {
- solrInputDocument.setField(n, new HashMap<String, String>() {
- {
- put("set", vals[0]);
- }
- });
- }
- } else if (vals.length > 1) {
- if (updateStrategy == UpdateStrategy.ADD) {
- solrInputDocument.setField(n, vals);
- } else {
- solrInputDocument.setField(n, new HashMap<String, String[]>() {
- {
- put("set", vals);
- }
- });
- }
- }
- }
- }
-
- /**
- * Options: SKIP, CONCATENATE_CONTENT, PARENT_CHILD. Default is "PARENT_CHILD".
- * If set to "SKIP", this will index only the main file and ignore all info
- * in the attachments. If set to "CONCATENATE_CONTENT", this will concatenate the
- * content extracted from the attachments into the main document and
- * then index the main document with the concatenated content _and_ the
- * main document's metadata (metadata from attachments will be thrown away).
- * If set to "PARENT_CHILD", this will index the attachments as children
- * of the parent document via Solr's parent-child relationship.
- */
- @Field
- public void setAttachmentStrategy(String attachmentStrategy) {
- this.attachmentStrategy = AttachmentStrategy.valueOf(attachmentStrategy);
- }
-
- @Field
- public void setUpdateStrategy(String updateStrategy) {
- this.updateStrategy = UpdateStrategy.valueOf(updateStrategy);
- }
-
- @Field
- public void setConnectionTimeout(int connectionTimeout) {
- this.connectionTimeout = connectionTimeout;
- }
-
- @Field
- public void setSocketTimeout(int socketTimeout) {
- this.socketTimeout = socketTimeout;
- }
-
- public int getCommitWithin() {
- return commitWithin;
- }
-
- @Field
- public void setCommitWithin(int commitWithin) {
- this.commitWithin = commitWithin;
- }
-
- /**
- * Specify the field in the first Metadata that should be
- * used as the id field for the document.
- *
- * @param idField
- */
- @Field
- public void setIdField(String idField) {
- this.idField = idField;
- }
-
- @Field
- public void setSolrCollection(String solrCollection) {
- this.solrCollection = solrCollection;
- }
-
- @Field
- public void setSolrUrls(List<String> solrUrls) {
- this.solrUrls = solrUrls;
- }
-
- @Field
- public void setSolrZkHosts(List<String> solrZkHosts) {
- this.solrZkHosts = solrZkHosts;
- }
-
- @Field
- public void setSolrZkChroot(String solrZkChroot) {
- this.solrZkChroot = solrZkChroot;
- }
-
- //TODO -- add other httpclient configurations??
- @Field
- public void setUserName(String userName) {
- httpClientFactory.setUserName(userName);
- }
-
- @Field
- public void setPassword(String password) {
- httpClientFactory.setPassword(password);
- }
-
- @Field
- public void setAuthScheme(String authScheme) {
- httpClientFactory.setAuthScheme(authScheme);
- }
-
- @Field
- public void setProxyHost(String proxyHost) {
- httpClientFactory.setProxyHost(proxyHost);
- }
-
- @Field
- public void setProxyPort(int proxyPort) {
- httpClientFactory.setProxyPort(proxyPort);
- }
-
- /**
- * If using the {@link AttachmentStrategy#PARENT_CHILD}, this is the field name
- * used to store the child documents. Note that we artificially flatten all embedded
- * documents, no matter how nested in the container document, into direct children
- * of the root document.
- *
- * @param embeddedFileFieldName
- */
- @Field
- public void setEmbeddedFileFieldName(String embeddedFileFieldName) {
- this.embeddedFileFieldName = embeddedFileFieldName;
- }
-
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- if (solrUrls == null || solrUrls.isEmpty()) {
- //TODO -- there's more that we need to pass through, including ssl etc.
- Http2SolrClient.Builder http2SolrClientBuilder = new Http2SolrClient.Builder();
- if (!StringUtils.isBlank(httpClientFactory.getUserName())) {
- http2SolrClientBuilder.withBasicAuthCredentials(httpClientFactory.getUserName(), httpClientFactory.getPassword());
- }
- http2SolrClientBuilder
- .withRequestTimeout(httpClientFactory.getRequestTimeout(), TimeUnit.MILLISECONDS)
- .withConnectionTimeout(connectionTimeout, TimeUnit.MILLISECONDS);
-
-
- Http2SolrClient http2SolrClient = http2SolrClientBuilder.build();
- solrClient = new CloudSolrClient.Builder(solrZkHosts, Optional.ofNullable(solrZkChroot))
- .withHttpClient(http2SolrClient)
- .build();
-
- } else {
- solrClient = new LBHttpSolrClient.Builder().withConnectionTimeout(connectionTimeout, TimeUnit.MILLISECONDS)
- .withSocketTimeout(socketTimeout, TimeUnit.MILLISECONDS).withHttpClient(httpClientFactory.build())
- .withBaseEndpoints(solrUrls.toArray(new String[]{})).build();
- }
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler)
- throws TikaConfigException {
- mustNotBeEmpty("solrCollection", this.solrCollection);
- mustNotBeEmpty("urlFieldName", this.idField);
- if ((this.solrUrls == null || this.solrUrls.isEmpty()) &&
- (this.solrZkHosts == null || this.solrZkHosts.isEmpty())) {
- throw new IllegalArgumentException(
- "expected either param solrUrls or param solrZkHosts, but neither was specified");
- }
- if (this.solrUrls != null && !this.solrUrls.isEmpty() && this.solrZkHosts != null &&
- !this.solrZkHosts.isEmpty()) {
- throw new IllegalArgumentException(
- "expected either param solrUrls or param solrZkHosts, but both were specified");
- }
- }
-
- public enum AttachmentStrategy {
- SEPARATE_DOCUMENTS, PARENT_CHILD,
- //anything else?
- }
-
- public enum UpdateStrategy {
- ADD, UPDATE_MUST_EXIST, UPDATE_MUST_NOT_EXIST,
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/SolrEmitterDevTest.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/SolrEmitterDevTest.java
deleted file mode 100644
index 1e68b1d4e..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/SolrEmitterDevTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.solr;
-
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.metadata.filter.FieldNameMappingFilter;
-import org.apache.tika.parser.ParseContext;
-
-/**
- * This is meant only for one off development tests with a locally
- * running instance of Solr. Please add unit tests to the
- * tika-integration-tests/tika-pipes-solr-integration-tests
- */
-@Disabled
-public class SolrEmitterDevTest {
-
- @Test
- public void oneOff() throws Exception {
- String core = "tika-example";
- String url = "http://localhost:8983/solr";
- String emitKey = "one-off-test-doc";
- SolrEmitter solrEmitter = new SolrEmitter();
- solrEmitter.setSolrUrls(Collections.singletonList(url));
- solrEmitter.setSolrCollection(core);
- solrEmitter.initialize(Collections.EMPTY_MAP);
-
- Metadata metadata = new Metadata();
- metadata.set(TikaCoreProperties.CREATED, new Date());
- metadata.set(TikaCoreProperties.TIKA_CONTENT, "the quick brown fox");
-
- Map<String, String> mappings = new HashMap();
- FieldNameMappingFilter filter = new FieldNameMappingFilter();
- mappings.put(TikaCoreProperties.CREATED.getName(), "created");
- mappings.put(TikaCoreProperties.TIKA_CONTENT.getName(), "content");
- filter.setMappings(mappings);
- filter.filter(metadata);
-
- solrEmitter.emit(emitKey, Collections.singletonList(metadata), new ParseContext());
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j2.xml b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j2.xml
deleted file mode 100644
index 5611a3556..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j2.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<Configuration status="WARN">
- <Appenders>
- <Console name="Console" target="SYSTEM_ERR">
- <PatternLayout pattern="%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n"/>
- </Console>
- </Appenders>
- <Loggers>
- <Root level="debug">
- <AppenderRef ref="Console"/>
- </Root>
- <Logger name="org.apache.http" level="INFO" additivity="false">
- <AppenderRef ref="Console"/>
- </Logger>
- </Loggers>
-</Configuration>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/tika-config-simple-emitter.xml b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/tika-config-simple-emitter.xml
deleted file mode 100644
index 5b14a5441..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/tika-config-simple-emitter.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <metadataFilters>
- <metadataFilter class="org.apache.tika.metadata.filter.FieldNameMappingFilter">
- <mappings>
- <mapping from="X-TIKA:content" to="content"/>
- <mapping from="X-TIKA:embedded_resource_path" to="embedded_path"/>
- <mapping from="Content-Length" to="length"/>
- <mapping from="dc:creator" to="creators"/>
- <mapping from="dc:title" to="title"/>
- </mappings>
- </metadataFilter>
- </metadataFilters>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.solr.SolrEmitter">
- <name>solr1</name>
- <url>http://localhost:8983/solr/tika-test</url>
- <attachmentStrategy>concatenate-content</attachmentStrategy>
- <contentField>content</contentField>
- <commitWithin>10</commitWithin>
- </emitter>
- <emitter class="org.apache.tika.pipes.emitter.solr.SolrEmitter">
- <name>solr2</name>
- <url>http://localhost:8983/solr/tika-test</url>
- <attachmentStrategy>parent-child</attachmentStrategy>
- <contentField>content</contentField>
- <commitWithin>10</commitWithin>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/pom.xml b/tika-pipes/tika-fetchers/pom.xml
deleted file mode 100644
index 999d269fc..000000000
--- a/tika-pipes/tika-fetchers/pom.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-pipes</artifactId>
- <version>4.0.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-fetchers</artifactId>
- <name>Apache Tika fetchers</name>
- <url>https://tika.apache.org/</url>
- <packaging>pom</packaging>
-
- <modules>
- <module>tika-fetcher-http</module>
- <module>tika-fetcher-s3</module>
- <module>tika-fetcher-gcs</module>
- <module>tika-fetcher-az-blob</module>
- <module>tika-fetcher-microsoft-graph</module>
- </modules>
-
- <dependencies>
-
- </dependencies>
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/pom.xml
deleted file mode 100644
index 2fd9153e3..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/pom.xml
+++ /dev/null
@@ -1,119 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-fetchers</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-fetcher-az-blob</artifactId>
- <name>Apache Tika Azure Blob fetcher</name>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.azure</groupId>
- <artifactId>azure-storage-blob</artifactId>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-serialization</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.fetcher.azblob</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/AZBlobFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/AZBlobFetcher.java
deleted file mode 100644
index 62f082b7f..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/AZBlobFetcher.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.azblob;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Path;
-import java.util.Map;
-
-import com.azure.storage.blob.BlobClient;
-import com.azure.storage.blob.BlobClientBuilder;
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.BlobServiceClient;
-import com.azure.storage.blob.BlobServiceClientBuilder;
-import com.azure.storage.blob.models.BlobProperties;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TemporaryResources;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
-import org.apache.tika.pipes.fetcher.azblob.config.AZBlobFetcherConfig;
-import org.apache.tika.utils.StringUtils;
-
-/**
- * Fetches files from Azure blob storage.
- * <p>
- * There are two modes:
- * 1) If you are only using one endpoint and one sas token and one container,
- * configure those in the config file. In this case, your fetchKey will
- * be the path in the container to the blob.
- * 2) If you have different endpoints or sas tokens or containers across
- * your requests, your fetchKey will be the complete SAS url pointing to the blob.
- */
-public class AZBlobFetcher extends AbstractFetcher implements Initializable {
- public AZBlobFetcher() {
-
- }
- public AZBlobFetcher(AZBlobFetcherConfig azBlobFetcherConfig) {
- setContainer(azBlobFetcherConfig.getContainer());
- setEndpoint(azBlobFetcherConfig.getEndpoint());
- setSasToken(azBlobFetcherConfig.getSasToken());
- setSpoolToTemp(azBlobFetcherConfig.isSpoolToTemp());
- setExtractUserMetadata(azBlobFetcherConfig.isExtractUserMetadata());
- }
-
- private static final Logger LOGGER = LoggerFactory.getLogger(AZBlobFetcher.class);
- private static String PREFIX = "az-blob";
- private String sasToken;
- private String container;
- private String endpoint;
- private BlobClientFactory blobClientFactory;
- private boolean extractUserMetadata = true;
- private BlobServiceClient blobServiceClient;
- private BlobContainerClient blobContainerClient;
- private boolean spoolToTemp = true;
-
- @Override
- public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws TikaException, IOException {
-
- LOGGER.debug("about to fetch fetchkey={} from endpoint ({})", fetchKey, endpoint);
-
- try {
- BlobClient blobClient = blobClientFactory.getClient(fetchKey);
-
- if (extractUserMetadata) {
- BlobProperties properties = blobClient.getProperties();
- if (properties.getMetadata() != null) {
- for (Map.Entry<String, String> e : properties
- .getMetadata()
- .entrySet()) {
- metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
- }
- }
- }
- if (!spoolToTemp) {
- return TikaInputStream.get(blobClient.openInputStream());
- } else {
- long start = System.currentTimeMillis();
- TemporaryResources tmpResources = new TemporaryResources();
- Path tmp = tmpResources.createTempFile();
- blobClient.downloadToFile(tmp.toRealPath().toString());
- TikaInputStream tis = TikaInputStream.get(tmp, metadata, tmpResources);
- long elapsed = System.currentTimeMillis() - start;
- LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
- return tis;
- }
- } catch (Exception e) {
- throw new IOException("az-blob storage exception", e);
- }
- }
-
- @Field
- public void setSpoolToTemp(boolean spoolToTemp) {
- this.spoolToTemp = spoolToTemp;
- }
-
- @Field
- public void setSasToken(String sasToken) {
- this.sasToken = sasToken;
- }
-
- @Field
- public void setEndpoint(String endpoint) {
- this.endpoint = endpoint;
- }
-
- @Field
- public void setContainer(String container) {
- this.container = container;
- }
-
- /**
- * Whether or not to extract user metadata from the blob object
- *
- * @param extractUserMetadata
- */
- @Field
- public void setExtractUserMetadata(boolean extractUserMetadata) {
- this.extractUserMetadata = extractUserMetadata;
- }
-
-
- /**
- * This initializes the az blob container client
- *
- * @param params params to use for initialization
- * @throws TikaConfigException
- */
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- if (!StringUtils.isBlank(sasToken)) {
- LOGGER.debug("Setting up immutable endpoint, token and container");
- blobClientFactory = new SingleBlobContainerFactory(endpoint, sasToken, container);
- } else {
- LOGGER.debug("Setting up blobclientfactory to recieve the full sas url for the blob");
- blobClientFactory = new SASUrlFactory();
- }
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
- //if the user has set one of these, they need to have set all of them
- if (!StringUtils.isBlank(this.sasToken) || !StringUtils.isBlank(this.endpoint) || !StringUtils.isBlank(this.container)) {
- mustNotBeEmpty("sasToken", this.sasToken);
- mustNotBeEmpty("endpoint", this.endpoint);
- mustNotBeEmpty("container", this.container);
- }
- }
-
- private interface BlobClientFactory {
- BlobClient getClient(String fetchKey);
- }
-
- private static class SingleBlobContainerFactory implements BlobClientFactory {
- private final BlobContainerClient blobContainerClient;
-
- private SingleBlobContainerFactory(String endpoint, String sasToken, String container) {
- //TODO -- allow authentication via other methods
- BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
- .endpoint(endpoint)
- .sasToken(sasToken)
- .buildClient();
- blobContainerClient = blobServiceClient.getBlobContainerClient(container);
- }
-
- @Override
- public BlobClient getClient(String fetchKey) {
- return blobContainerClient.getBlobClient(fetchKey);
- }
- }
-
- private static class SASUrlFactory implements BlobClientFactory {
-
- @Override
- public BlobClient getClient(String fetchKey) {
- return new BlobClientBuilder()
- .connectionString(fetchKey)
- .buildClient();
- }
- }
-}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/config/AZBlobFetcherConfig.java b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/config/AZBlobFetcherConfig.java
deleted file mode 100644
index e29e62319..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/config/AZBlobFetcherConfig.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.azblob.config;
-
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
-
-public class AZBlobFetcherConfig extends AbstractConfig {
- private boolean spoolToTemp;
- private String sasToken;
- private String endpoint;
- private String container;
- private boolean extractUserMetadata;
-
- public boolean isSpoolToTemp() {
- return spoolToTemp;
- }
-
- public AZBlobFetcherConfig setSpoolToTemp(boolean spoolToTemp) {
- this.spoolToTemp = spoolToTemp;
- return this;
- }
-
- public String getSasToken() {
- return sasToken;
- }
-
- public AZBlobFetcherConfig setSasToken(String sasToken) {
- this.sasToken = sasToken;
- return this;
- }
-
- public String getEndpoint() {
- return endpoint;
- }
-
- public AZBlobFetcherConfig setEndpoint(String endpoint) {
- this.endpoint = endpoint;
- return this;
- }
-
- public String getContainer() {
- return container;
- }
-
- public AZBlobFetcherConfig setContainer(String container) {
- this.container = container;
- return this;
- }
-
- public boolean isExtractUserMetadata() {
- return extractUserMetadata;
- }
-
- public AZBlobFetcherConfig setExtractUserMetadata(boolean extractUserMetadata) {
- this.extractUserMetadata = extractUserMetadata;
- return this;
- }
-}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/java/org/apache/tika/pipes/fetcher/azblob/TestAZBlobFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/java/org/apache/tika/pipes/fetcher/azblob/TestAZBlobFetcher.java
deleted file mode 100644
index bbfc63c65..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/java/org/apache/tika/pipes/fetcher/azblob/TestAZBlobFetcher.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.azblob;
-
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Paths;
-import java.util.List;
-
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.TikaTest;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.Fetcher;
-import org.apache.tika.pipes.core.fetcher.FetcherManager;
-import org.apache.tika.serialization.JsonMetadataList;
-
-@Disabled("write actual unit tests")
-public class TestAZBlobFetcher extends TikaTest {
-
- private static final String FETCH_STRING = "something-or-other/test-out.json";
-
- @Test
- public void testConfig() throws Exception {
- FetcherManager fetcherManager = FetcherManager.load(Paths.get(this
- .getClass()
- .getResource("/tika-config-az-blob.xml")
- .toURI()));
- Fetcher fetcher = fetcherManager.getFetcher("az-blob");
- List<Metadata> metadataList = null;
- try (Reader reader = new BufferedReader(new InputStreamReader(fetcher.fetch(FETCH_STRING, new Metadata(), new ParseContext()), StandardCharsets.UTF_8))) {
- metadataList = JsonMetadataList.fromJson(reader);
- }
- debug(metadataList);
- }
-}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/resources/tika-config-az-blob.xml b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/resources/tika-config-az-blob.xml
deleted file mode 100644
index 2aa6ba9a5..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/resources/tika-config-az-blob.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<properties>
- <fetchers>
- <fetcher class="org.apache.tika.pipes.fetcher.azblob.AZBlobFetcher">
- <name>az-blob</name>
- <!-- Either configure these three and send in the path for the blob OR
- do not configure any of these and send in the full SAS url for the blob
- as the fetchkey-->
- <endpoint></endpoint>
- <container></container>
- <sasToken></sasToken>
- </fetcher>
- </fetchers>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-gcs/pom.xml
deleted file mode 100644
index cbf4555e3..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/pom.xml
+++ /dev/null
@@ -1,106 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-fetchers</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-fetcher-gcs</artifactId>
- <name>Apache Tika Google Cloud Storage fetcher</name>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.cloud</groupId>
- <artifactId>google-cloud-storage</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.fetcher.gcs</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java
deleted file mode 100644
index 42e7cf486..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.gcs;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Path;
-import java.util.Map;
-
-import com.google.cloud.storage.Blob;
-import com.google.cloud.storage.BlobId;
-import com.google.cloud.storage.Storage;
-import com.google.cloud.storage.StorageOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TemporaryResources;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
-import org.apache.tika.pipes.fetcher.gcs.config.GCSFetcherConfig;
-
-/**
- * Fetches files from google cloud storage. Must set projectId and bucket via the config.
- */
-public class GCSFetcher extends AbstractFetcher implements Initializable {
- public GCSFetcher() {
-
- }
- public GCSFetcher(GCSFetcherConfig gcsFetcherConfig) {
- setBucket(gcsFetcherConfig.getBucket());
- setProjectId(gcsFetcherConfig.getProjectId());
- setSpoolToTemp(gcsFetcherConfig.isSpoolToTemp());
- setExtractUserMetadata(gcsFetcherConfig.isExtractUserMetadata());
- }
- private static String PREFIX = "gcs";
- private static final Logger LOGGER = LoggerFactory.getLogger(GCSFetcher.class);
- private String projectId;
- private String bucket;
- private boolean extractUserMetadata = true;
- private Storage storage;
- private boolean spoolToTemp = true;
-
- @Override
- public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws TikaException, IOException {
-
- LOGGER.debug("about to fetch fetchkey={} from bucket ({})", fetchKey, bucket);
-
- try {
- Blob blob = storage.get(BlobId.of(bucket, fetchKey));
-
- if (extractUserMetadata) {
- if (blob.getMetadata() != null) {
- for (Map.Entry<String, String> e : blob.getMetadata().entrySet()) {
- metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
- }
- }
- }
- if (!spoolToTemp) {
- return TikaInputStream.get(blob.getContent());
- } else {
- long start = System.currentTimeMillis();
- TemporaryResources tmpResources = new TemporaryResources();
- Path tmp = tmpResources.createTempFile();
- blob.downloadTo(tmp);
- TikaInputStream tis = TikaInputStream.get(tmp, metadata, tmpResources);
- long elapsed = System.currentTimeMillis() - start;
- LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
- return tis;
- }
- } catch (Exception e) {
- throw new IOException("gcs storage exception", e);
- }
- }
-
- @Field
- public void setSpoolToTemp(boolean spoolToTemp) {
- this.spoolToTemp = spoolToTemp;
- }
-
- @Field
- public void setProjectId(String projectId) {
- this.projectId = projectId;
- }
-
- @Field
- public void setBucket(String bucket) {
- this.bucket = bucket;
- }
-
- /**
- * Whether or not to extract user metadata from the S3Object
- *
- * @param extractUserMetadata
- */
- @Field
- public void setExtractUserMetadata(boolean extractUserMetadata) {
- this.extractUserMetadata = extractUserMetadata;
- }
-
- //TODO: parameterize extracting other blob metadata, eg. md5, crc, etc.
-
- /**
- * This initializes the gcs storage client.
- *
- * @param params params to use for initialization
- * @throws TikaConfigException
- */
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- //params have already been set...ignore them
- //TODO -- add other params to the builder as needed
- storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler)
- throws TikaConfigException {
- mustNotBeEmpty("bucket", this.bucket);
- mustNotBeEmpty("projectId", this.projectId);
- }
-}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/config/GCSFetcherConfig.java b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/config/GCSFetcherConfig.java
deleted file mode 100644
index 6a808f1ec..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/config/GCSFetcherConfig.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.gcs.config;
-
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
-
-public class GCSFetcherConfig extends AbstractConfig {
- private boolean spoolToTemp;
- private String projectId;
- private String bucket;
- private boolean extractUserMetadata;
-
- public boolean isSpoolToTemp() {
- return spoolToTemp;
- }
-
- public GCSFetcherConfig setSpoolToTemp(boolean spoolToTemp) {
- this.spoolToTemp = spoolToTemp;
- return this;
- }
-
- public String getProjectId() {
- return projectId;
- }
-
- public GCSFetcherConfig setProjectId(String projectId) {
- this.projectId = projectId;
- return this;
- }
-
- public String getBucket() {
- return bucket;
- }
-
- public GCSFetcherConfig setBucket(String bucket) {
- this.bucket = bucket;
- return this;
- }
-
- public boolean isExtractUserMetadata() {
- return extractUserMetadata;
- }
-
- public GCSFetcherConfig setExtractUserMetadata(boolean extractUserMetadata) {
- this.extractUserMetadata = extractUserMetadata;
- return this;
- }
-}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/java/org/apache/tika/pipes/fetcher/s3/TestGCSFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/java/org/apache/tika/pipes/fetcher/s3/TestGCSFetcher.java
deleted file mode 100644
index 3844337f3..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/java/org/apache/tika/pipes/fetcher/s3/TestGCSFetcher.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.s3;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
-
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.Fetcher;
-import org.apache.tika.pipes.core.fetcher.FetcherManager;
-
-@Disabled("write actual unit tests")
-public class TestGCSFetcher {
-
- private static final String FETCH_STRING = "testExtraSpaces.pdf";
-
- @TempDir
- private static Path TEMP_DIR;
- private static Path outputFile;
-
- @BeforeAll
- public static void setUp() throws Exception {
- outputFile = Files.createTempFile(TEMP_DIR, "tika-test", ".pdf");
- }
-
-
- @Test
- public void testConfig() throws Exception {
- FetcherManager fetcherManager = FetcherManager.load(
- Paths.get(this.getClass().getResource("/tika-config-gcs.xml").toURI()));
- Fetcher fetcher = fetcherManager.getFetcher("gcs");
- Metadata metadata = new Metadata();
- try (InputStream is = fetcher.fetch(FETCH_STRING, metadata, new ParseContext())) {
- Files.copy(is, outputFile, StandardCopyOption.REPLACE_EXISTING);
- }
- assertEquals(20743, Files.size(outputFile));
- }
-}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/resources/tika-config-gcs.xml b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/resources/tika-config-gcs.xml
deleted file mode 100644
index 7e9ba6817..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/resources/tika-config-gcs.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<properties>
- <fetchers>
- <fetcher class="org.apache.tika.pipes.fetcher.gcs.GCSFetcher">
- <name>gcs</name>
- <projectId>My First Project</projectId>
- <bucket>tika-tallison-test-bucket</bucket>
- </fetcher>
- </fetchers>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
deleted file mode 100644
index 00c27fc18..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
+++ /dev/null
@@ -1,139 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-fetchers</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-fetcher-http</artifactId>
- <name>Apache Tika http fetcher</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-httpclient-commons</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.nimbusds</groupId>
- <artifactId>nimbus-jose-jwt</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.fetcher.http</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
deleted file mode 100644
index ee1953cb7..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
+++ /dev/null
@@ -1,690 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.http;
-
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.InetAddress;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.security.PrivateKey;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.nimbusds.jose.JOSEException;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
-import org.apache.http.ConnectionClosedException;
-import org.apache.http.Header;
-import org.apache.http.HttpConnection;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpInetConnection;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.impl.conn.ConnectionShutdownException;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.client.HttpClientFactory;
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.exception.TikaTimeoutException;
-import org.apache.tika.io.TemporaryResources;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.Property;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
-import org.apache.tika.pipes.core.fetcher.RangeFetcher;
-import org.apache.tika.pipes.core.fetcher.config.FetcherConfigContainer;
-import org.apache.tika.pipes.fetcher.http.config.HttpFetcherConfig;
-import org.apache.tika.pipes.fetcher.http.config.HttpHeaders;
-import org.apache.tika.pipes.fetcher.http.jwt.JwtGenerator;
-import org.apache.tika.pipes.fetcher.http.jwt.JwtPrivateKeyCreds;
-import org.apache.tika.pipes.fetcher.http.jwt.JwtSecretCreds;
-import org.apache.tika.utils.StringUtils;
-
-/**
- * Based on Apache httpclient
- */
-public class HttpFetcher extends AbstractFetcher implements Initializable, RangeFetcher {
- public HttpFetcher() {
-
- }
-
- private static final ObjectMapper OM = new ObjectMapper();
- private HttpFetcherConfig httpFetcherConfig = new HttpFetcherConfig();
- private HttpClientFactory httpClientFactory = new HttpClientFactory();
-
- public HttpFetcher(HttpFetcherConfig httpFetcherConfig) {
- this.httpFetcherConfig = httpFetcherConfig;
- }
-
- public static String HTTP_HEADER_PREFIX = "http-header:";
-
- public static String HTTP_FETCH_PREFIX = "http-connection:";
-
- /**
- * http status code
- */
- public static Property HTTP_STATUS_CODE = Property.externalInteger(HTTP_HEADER_PREFIX + "status-code");
- /**
- * Number of redirects
- */
- public static Property HTTP_NUM_REDIRECTS = Property.externalInteger(HTTP_FETCH_PREFIX + "num-redirects");
-
- /**
- * If there were redirects, this captures the final URL visited
- */
- public static Property HTTP_TARGET_URL = Property.externalText(HTTP_FETCH_PREFIX + "target-url");
-
- public static Property HTTP_TARGET_IP_ADDRESS = Property.externalText(HTTP_FETCH_PREFIX + "target-ip-address");
-
- public static Property HTTP_FETCH_TRUNCATED = Property.externalBoolean(HTTP_FETCH_PREFIX + "fetch-truncated");
-
- public static Property HTTP_CONTENT_ENCODING = Property.externalText(HTTP_HEADER_PREFIX + "content-encoding");
-
- public static Property HTTP_CONTENT_TYPE = Property.externalText(HTTP_HEADER_PREFIX + "content-type");
-
- private static String USER_AGENT = "User-Agent";
-
- Logger LOG = LoggerFactory.getLogger(HttpFetcher.class);
-
- private HttpClient httpClient;
- //back-off client that disables compression
- private HttpClient noCompressHttpClient;
- private int maxRedirects = 10;
- //overall timeout in milliseconds
- private long overallTimeout = -1;
-
- private long maxSpoolSize = -1;
-
- //max string length to read from a result if the
- //status code was not in the 200 range
- private int maxErrMsgSize = 10000;
-
- //httpHeaders to capture in the metadata
- private Set<String> httpHeaders = new HashSet<>();
-
- //httpRequestHeaders to add to all outgoing http requests
- private Set<String> httpRequestHeaders = new HashSet<>();
-
- //When making the request, what User-Agent is sent.
- //By default httpclient adds e.g. "Apache-HttpClient/4.5.13 (Java/x.y.z)"
- private String userAgent = null;
-
- JwtGenerator jwtGenerator;
-
- @Override
- public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws IOException, TikaException {
- HttpFetcherConfig additionalHttpFetcherConfig = getAdditionalHttpFetcherConfig(parseContext);
- HttpGet get = new HttpGet(fetchKey);
- RequestConfig requestConfig = RequestConfig
- .custom()
- .setMaxRedirects(httpFetcherConfig.getMaxRedirects())
- .setRedirectsEnabled(httpFetcherConfig.getMaxRedirects() > 0)
- .build();
- get.setConfig(requestConfig);
- setHttpRequestHeaders(metadata, get);
- putAdditionalHeadersOnRequest(additionalHttpFetcherConfig, get);
- return execute(get, metadata, httpClient, true);
- }
-
- private void setHttpRequestHeaders(Metadata metadata, HttpGet get) {
- if (!StringUtils.isBlank(userAgent)) {
- get.setHeader(USER_AGENT, userAgent);
- }
- // Add the headers from the Fetcher configuration.
- if (httpRequestHeaders != null) {
- for (String httpRequestHeader : httpRequestHeaders) {
- parseHeaderAndPutOnRequest(get, httpRequestHeader);
- }
- }
- // Additionally, headers can be specified per-fetch via the metadata.
- String[] httpRequestHeaders = metadata.getValues("httpRequestHeaders");
- if (httpRequestHeaders != null) {
- for (String httpRequestHeader : httpRequestHeaders) {
- parseHeaderAndPutOnRequest(get, httpRequestHeader);
- }
- }
- }
-
- private static void parseHeaderAndPutOnRequest(HttpGet get, String httpRequestHeader) {
- String[] parts = httpRequestHeader
- .trim().split(":", 2);
- if (parts.length >= 2) {
- String key = parts[0].trim();
- String value = parts[1].trim();
- get.setHeader(key, value);
- }
- }
-
- private HttpFetcherConfig getAdditionalHttpFetcherConfig(ParseContext parseContext) throws JsonProcessingException {
- HttpFetcherConfig additionalHttpFetcherConfig = null;
- FetcherConfigContainer fetcherConfigContainer = parseContext.get(FetcherConfigContainer.class);
- if (fetcherConfigContainer != null) {
- additionalHttpFetcherConfig = OM.readValue(fetcherConfigContainer.getJson(), HttpFetcherConfig.class);
- }
- return additionalHttpFetcherConfig;
- }
-
- @Override
- public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata,
- ParseContext parseContext) throws IOException, TikaException {
- HttpFetcherConfig additionalHttpFetcherConfig = getAdditionalHttpFetcherConfig(parseContext);
- HttpGet get = new HttpGet(fetchKey);
- putAdditionalHeadersOnRequest(additionalHttpFetcherConfig, get);
-
- get.setHeader("Range", "bytes=" + startRange + "-" + endRange);
- return execute(get, metadata, httpClient, true);
- }
-
- private void putAdditionalHeadersOnRequest(HttpFetcherConfig additionalFetcherConfig, HttpGet httpGet) throws TikaException {
- if (!StringUtils.isBlank(httpFetcherConfig.getUserAgent())) {
- httpGet.setHeader(USER_AGENT, httpFetcherConfig.getUserAgent());
- }
- if (additionalFetcherConfig != null && additionalFetcherConfig.getHttpRequestHeaders() != null) {
- additionalFetcherConfig.getHttpRequestHeaders()
- .getHeaders()
- .forEach(httpGet::addHeader);
- }
- if (httpFetcherConfig.getHttpRequestHeaders() != null) {
- httpFetcherConfig.getHttpRequestHeaders()
- .getHeaders()
- .forEach(httpGet::addHeader);
- }
- if (jwtGenerator != null) {
- try {
- httpGet.setHeader("Authorization", "Bearer " + jwtGenerator.jwt());
- } catch (JOSEException e) {
... 37346 lines suppressed ...
(tika) 02/04: TIKA-4334: replace pipes and server with newer apis
Posted by nd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ndipiazza pushed a commit to branch TIKA-4334
in repository https://gitbox.apache.org/repos/asf/tika.git
commit af31f994fb4f0031db425f203cea5e7838343f4c
Author: Nicholas DiPiazza <nd...@apache.org>
AuthorDate: Mon Sep 29 11:24:53 2025 -0500
TIKA-4334: replace pipes and server with newer apis
---
pom.xml | 1 -
tika-app/pom.xml | 3 +-
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 39 --
tika-bom/pom.xml | 6 -
tika-parent/pom.xml | 20 +
tika-pipes/pom.xml | 31 +-
tika-pipes/tika-pipes-cli/pom.xml | 77 ++-
.../cli/googledrive/GoogleDriveFetcherCli.java | 2 +-
.../apache/tika/pipes/cli/http/HttpFetcherCli.java | 3 +-
.../microsoftgraph/MicrosoftGraphFetcherCli.java | 3 +-
.../org/apache/tika/pipes/cli/s3/S3FetcherCli.java | 3 +-
.../org/apache/tika/pipes/core/HandlerConfig.java | 192 ++++++
.../tika/pipes/core/parser/ParseService.java | 2 +-
.../org/apache/tika/pipes/core/util/OSUtils.java | 258 --------
.../tika-emitter-file-system/pom.xml | 2 +-
.../fetchers/filesystem/FileSystemEmitter.java | 3 +-
.../src/main/resources/plugin.properties | 2 +-
.../tika-pipes-emitters/tika-emitters-core/pom.xml | 2 +-
.../tika-fetcher-atlassian-jwt/pom.xml | 4 +-
.../atlassianjwt/AtlassianJwtGenerator.java | 3 +-
.../src/main/resources/plugin.properties | 2 +-
.../tika-fetcher-file-system/pom.xml | 2 +-
.../src/main/resources/plugin.properties | 2 +-
.../tika-fetcher-google-drive/pom.xml | 71 ++-
.../src/main/resources/plugin.properties | 2 +-
.../tika-pipes-fetchers/tika-fetcher-http/pom.xml | 4 +-
.../src/main/resources/plugin.properties | 2 +-
.../tika-fetcher-microsoft-graph/pom.xml | 32 +-
.../src/main/resources/plugin.properties | 2 +-
.../tika-pipes-fetchers/tika-fetcher-s3/pom.xml | 21 +-
.../src/main/resources/plugin.properties | 2 +-
.../tika-pipes-fetchers/tika-fetchers-core/pom.xml | 31 +
tika-pipes/tika-pipes-grpc/docker-build/Dockerfile | 2 +-
tika-pipes/tika-pipes-grpc/pom.xml | 35 +-
.../pipes/grpc/TikaGrpcServerFetchersCrudTest.java | 2 +-
.../pipes/grpc/TikaGrpcServerJsonSchemaTest.java | 2 +-
.../tika/pipes/grpc/TikaGrpcServerPipeJobTest.java | 2 +-
.../tika-pipe-iterator-csv/pom.xml | 4 +-
.../src/main/resources/plugin.properties | 2 +-
.../tika-pipe-iterators-core/pom.xml | 31 +
tika-pipes/tika-pipes-proto/pom.xml | 60 +-
tika-server-old/tika-server-client/pom.xml | 124 ----
.../org/apache/tika/server/client/TikaClient.java | 69 ---
.../apache/tika/server/client/TikaClientCLI.java | 148 -----
.../server/client/TikaClientConfigException.java | 30 -
.../tika/server/client/TikaEmitterResult.java | 54 --
.../tika/server/client/TikaPipesHttpClient.java | 168 -----
.../tika/server/client/TikaServerClientConfig.java | 115 ----
.../org/apache/tika/server/client/TestBasic.java | 58 --
.../resources/tika-config-simple-fs-emitter.xml | 106 ----
tika-server-old/tika-server-core/pom.xml | 334 ----------
.../server/core/CompositeParseContextConfig.java | 42 --
.../tika/server/core/FetcherStreamFactory.java | 140 -----
.../org/apache/tika/server/core/HTMLHelper.java | 66 --
.../tika/server/core/InputStreamFactory.java | 38 --
.../tika/server/core/ParseContextConfig.java | 56 --
.../server/core/ProduceTypeResourceComparator.java | 145 -----
.../org/apache/tika/server/core/ServerStatus.java | 156 -----
.../tika/server/core/ServerStatusResource.java | 23 -
.../tika/server/core/ServerStatusWatcher.java | 224 -------
.../apache/tika/server/core/TikaLoggingFilter.java | 52 --
.../org/apache/tika/server/core/TikaServerCli.java | 231 -------
.../apache/tika/server/core/TikaServerConfig.java | 666 --------------------
.../core/TikaServerParseExceptionMapper.java | 99 ---
.../apache/tika/server/core/TikaServerProcess.java | 514 ---------------
.../tika/server/core/TikaServerWatchDog.java | 459 --------------
.../org/apache/tika/server/core/TlsConfig.java | 160 -----
.../apache/tika/server/core/WatchDogResult.java | 47 --
.../server/core/config/DocumentSelectorConfig.java | 46 --
.../server/core/config/PasswordProviderConfig.java | 53 --
.../tika/server/core/config/TimeoutConfig.java | 37 --
.../tika/server/core/resource/AsyncRequest.java | 33 -
.../tika/server/core/resource/AsyncResource.java | 193 ------
.../server/core/resource/LanguageResource.java | 70 ---
.../server/core/resource/MetadataResource.java | 148 -----
.../tika/server/core/resource/PipesResource.java | 161 -----
.../core/resource/RecursiveMetadataResource.java | 181 ------
.../tika/server/core/resource/TikaDetectors.java | 140 -----
.../tika/server/core/resource/TikaMimeTypes.java | 418 -------------
.../tika/server/core/resource/TikaParsers.java | 260 --------
.../tika/server/core/resource/TikaResource.java | 684 --------------------
.../server/core/resource/TikaServerResource.java | 23 -
.../server/core/resource/TikaServerStatus.java | 47 --
.../tika/server/core/resource/TikaVersion.java | 39 --
.../tika/server/core/resource/TikaWelcome.java | 234 -------
.../server/core/resource/TranslateResource.java | 123 ----
.../server/core/resource/UnpackerResource.java | 286 ---------
.../server/core/writer/CSVMessageBodyWriter.java | 71 ---
.../server/core/writer/JSONMessageBodyWriter.java | 58 --
.../tika/server/core/writer/JSONObjWriter.java | 63 --
.../core/writer/MetadataListMessageBodyWriter.java | 61 --
.../apache/tika/server/core/writer/TarWriter.java | 68 --
.../server/core/writer/TextMessageBodyWriter.java | 75 ---
.../tika/server/core/writer/TikaServerWriter.java | 26 -
.../apache/tika/server/core/writer/ZipWriter.java | 88 ---
.../org.apache.tika.server.core.ParseContextConfig | 17 -
.../main/resources/tika-server-config-default.xml | 93 ---
.../src/main/resources/tikaserver-template.html | 32 -
.../main/resources/tikaserver-version.properties | 15 -
.../org/apache/tika/server/core/CXFTestBase.java | 310 ----------
.../tika/server/core/IntegrationTestBase.java | 148 -----
.../tika/server/core/LanguageResourceTest.java | 114 ----
.../server/core/RecursiveMetadataResourceTest.java | 140 -----
.../apache/tika/server/core/ServerStatusTest.java | 108 ----
.../apache/tika/server/core/StackTraceOffTest.java | 163 -----
.../apache/tika/server/core/StackTraceTest.java | 159 -----
.../apache/tika/server/core/TikaMimeTypesTest.java | 103 ----
.../org/apache/tika/server/core/TikaPipesTest.java | 307 ---------
.../tika/server/core/TikaResourceFetcherTest.java | 144 -----
.../core/TikaResourceMetadataFilterTest.java | 80 ---
.../tika/server/core/TikaResourceNoStackTest.java | 99 ---
.../apache/tika/server/core/TikaResourceTest.java | 284 ---------
.../core/TikaServerAsyncIntegrationTest.java | 184 ------
.../tika/server/core/TikaServerConfigTest.java | 153 -----
.../server/core/TikaServerIntegrationTest.java | 665 --------------------
.../core/TikaServerPipesIntegrationTest.java | 243 --------
.../tika/server/core/TikaServerStatusTest.java | 83 ---
.../apache/tika/server/core/TikaVersionTest.java | 56 --
.../apache/tika/server/core/TikaWelcomeTest.java | 111 ----
.../tika/server/core/TranslateResourceTest.java | 89 ---
.../resources/configs/metadata-filter-include.xml | 28 -
.../configs/tika-config-server-badjvmargs.xml | 31 -
.../resources/configs/tika-config-server-basic.xml | 32 -
.../configs/tika-config-server-emitter.xml | 33 -
.../tika-config-server-fetcher-template.xml | 34 -
.../tika-config-server-fetchers-emitters.xml | 43 --
.../configs/tika-config-server-timeout-10000.xml | 28 -
.../tika-config-server-tls-one-way-template.xml | 39 --
.../tika-config-server-tls-two-way-template.xml | 42 --
.../resources/configs/tika-config-server-tls.xml | 41 --
.../test/resources/configs/tika-config-server.xml | 32 -
.../src/test/resources/logging/log4j2_forked.xml | 45 --
.../src/test/resources/ssl-keys/README.txt | 28 -
.../resources/ssl-keys/tika-client-keystore.p12 | Bin 2505 -> 0 bytes
.../resources/ssl-keys/tika-client-truststore.p12 | Bin 3429 -> 0 bytes
.../src/test/resources/ssl-keys/tika-client.crt | Bin 789 -> 0 bytes
.../resources/ssl-keys/tika-server-keystore.p12 | Bin 2505 -> 0 bytes
.../resources/ssl-keys/tika-server-truststore.p12 | Bin 3429 -> 0 bytes
.../src/test/resources/ssl-keys/tika-server.crt | Bin 789 -> 0 bytes
.../src/test/resources/test-documents/english.txt | 1 -
.../src/test/resources/test-documents/french.txt | 1 -
.../mock/encrypted_document_exception.xml | 23 -
.../resources/test-documents/mock/fake_oom.xml | 24 -
.../test-documents/mock/heavy_hang_100.xml | 25 -
.../test-documents/mock/heavy_hang_30000.xml | 25 -
.../resources/test-documents/mock/hello_world.xml | 26 -
.../test-documents/mock/hello_world_long.xml | 30 -
.../resources/test-documents/mock/null_pointer.xml | 25 -
.../resources/test-documents/mock/real_oom.xml | 24 -
.../resources/test-documents/mock/system_exit.xml | 25 -
.../test-documents/mock/testStaticStdOutErr.xml | 76 ---
.../test-documents/mock/testStdOutErr.xml | 76 ---
.../test-documents/mock/thread_interrupt.xml | 25 -
.../mock/\344\270\255\346\226\207.xml" | 27 -
.../test/resources/test-documents/testDigilite.fdf | 1 -
tika-server-old/tika-server-eval/pom.xml | 132 ----
.../apache/tika/server/eval/TikaEvalResource.java | 190 ------
...he.tika.server.core.resource.TikaServerResource | 15 -
.../tika/server/eval/TikaEvalResourceTest.java | 170 -----
.../tika/server/controller/AsyncController.java | 182 ------
.../tika/server/controller/DetectorController.java | 84 ---
.../controller/InformationServicesController.java | 282 ---------
.../controller/LanguageResourceController.java | 142 -----
.../tika/server/controller/MetadataController.java | 177 ------
.../tika/server/controller/PipesController.java | 194 ------
.../RecursiveMetadataAndContentController.java | 240 -------
.../server/controller/ServerStatusController.java | 101 ---
.../tika/server/controller/TikaController.java | 292 ---------
.../controller/TranslateResourceController.java | 118 ----
.../controller/UnpackResourceController.java | 249 --------
tika-server-old/tika-server-standard/assembly.xml | 46 --
.../tika-server-standard/bin/init.d/tika | 78 ---
.../bin/install_tika_service.sh | 363 -----------
tika-server-old/tika-server-standard/bin/tika | 567 -----------------
.../tika-server-standard/bin/tika.in.sh | 80 ---
tika-server-old/tika-server-standard/pom.xml | 313 ----------
.../server/standard/config/PDFServerConfig.java | 70 ---
.../standard/config/TesseractServerConfig.java | 70 ---
.../standard/resource/XMPMetadataResource.java | 66 --
.../standard/writer/XMPMessageBodyWriter.java | 67 --
.../services/org.apache.tika.parser.Parser | 16 -
.../org.apache.tika.server.core.ParseContextConfig | 16 -
...he.tika.server.core.resource.TikaServerResource | 15 -
...apache.tika.server.core.writer.TikaServerWriter | 15 -
.../tika/server/standard/DetectorResourceTest.java | 104 ----
.../apache/tika/server/standard/FetcherTest.java | 103 ----
.../server/standard/JsonMaxFieldLengthTest.java | 88 ---
.../tika/server/standard/MetadataResourceTest.java | 243 --------
.../server/standard/OpenNLPMetadataFilterTest.java | 115 ----
.../standard/OptimaizeMetadataFilterTest.java | 115 ----
.../standard/RecursiveMetadataFilterTest.java | 105 ----
.../standard/RecursiveMetadataResourceTest.java | 563 -----------------
.../tika/server/standard/TikaDetectorsTest.java | 153 -----
.../tika/server/standard/TikaMimeTypesTest.java | 89 ---
.../tika/server/standard/TikaParsersTest.java | 205 ------
.../apache/tika/server/standard/TikaPipesTest.java | 340 ----------
.../tika/server/standard/TikaResourceTest.java | 686 ---------------------
.../tika/server/standard/UnpackerResourceTest.java | 314 ----------
.../standard/UnpackerResourceWithConfigTest.java | 179 ------
.../test/resources/config/TIKA-3137-include.xml | 38 --
.../config/tika-config-for-server-tests.xml | 29 -
.../src/test/resources/config/tika-config-json.xml | 20 -
.../tika-config-langdetect-opennlp-filter.xml | 32 -
.../tika-config-langdetect-optimaize-filter.xml | 32 -
.../test/resources/config/tika-config-unpacker.xml | 32 -
.../resources/config/tika-config-url-fetcher.xml | 31 -
.../src/test/resources/test-documents/2exe.docx | Bin 715333 -> 0 bytes
.../src/test/resources/test-documents/2pic.doc | Bin 4339712 -> 0 bytes
.../src/test/resources/test-documents/2pic.docx | Bin 883427 -> 0 bytes
.../test-documents/CDEC_WEATHER_2010_03_02 | 98 ---
.../src/test/resources/test-documents/Doc1_ole.doc | Bin 89600 -> 0 bytes
.../src/test/resources/test-documents/foo.csv | 4 -
.../src/test/resources/test-documents/password.xls | Bin 22528 -> 0 bytes
.../src/test/resources/test-documents/pic.xls | Bin 593920 -> 0 bytes
.../src/test/resources/test-documents/pic.xlsx | Bin 580188 -> 0 bytes
.../src/test/resources/test-documents/test.doc | Bin 9216 -> 0 bytes
.../test-documents/testColorRendering.pdf | Bin 1794 -> 0 bytes
.../test/resources/test-documents/testHTML.html | 28 -
.../src/test/resources/test-documents/testOCR.pdf | Bin 41936 -> 0 bytes
.../test-documents/testPDFTwoTextBoxes.pdf | Bin 57100 -> 0 bytes
.../test-documents/testPDF_XFA_govdocs1_258578.pdf | Bin 168176 -> 0 bytes
.../test-documents/testPassword4Spaces.pdf | Bin 18285 -> 0 bytes
.../testRTF_npeFromWMFInTikaServer.rtf | 235 -------
.../test-documents/testUnicodePassword.pdf | Bin 18849 -> 0 bytes
.../test-documents/test_recursive_embedded.docx | Bin 27082 -> 0 bytes
{tika-server-old => tika-server}/README.md | 0
{tika-server-old => tika-server}/pom.xml | 4 -
.../tika-server-spring/pom.xml | 58 +-
.../apache/tika/server/TikaServerApplication.java | 0
.../apache/tika/server/component/ServerStatus.java | 207 +++++++
.../tika/server/controller/AsyncController.java | 35 +-
.../tika/server/controller/DetectorController.java | 76 ++-
.../tika/server/controller/EvalController.java | 49 ++
.../controller/InformationServicesController.java | 73 +++
.../controller/LanguageResourceController.java | 58 ++
.../tika/server/controller/MetadataController.java | 55 ++
.../tika/server/controller/PipesController.java | 35 +-
.../RecursiveMetadataAndContentController.java | 61 ++
.../server/controller/ServerStatusController.java | 33 +-
.../tika/server/controller/TikaController.java | 70 +++
.../controller/TranslateResourceController.java | 63 ++
.../controller/UnpackResourceController.java | 49 ++
.../server/controller/XmpMetadataController.java | 52 ++
.../org/apache/tika/server/util/TikaResource.java | 119 ++++
.../main/resources/api/tika-server-openapi.yaml | 373 ++++++++++-
.../src/main/resources/application.properties | 0
.../apache/tika/server/IntegrationTestBase.java | 48 ++
.../controller/AsyncControllerIntegrationTest.java | 25 +-
.../DetectorControllerIntegrationTest.java | 318 ++++++++++
...formationServicesControllerIntegrationTest.java | 28 +-
.../LanguageResourceControllerIntegrationTest.java | 24 +-
.../MetadataControllerIntegrationTest.java | 39 ++
.../controller/PipesControllerIntegrationTest.java | 27 +-
...etadataAndContentControllerIntegrationTest.java | 38 ++
.../ServerStatusControllerIntegrationTest.java | 24 +-
.../controller/TikaControllerIntegrationTest.java | 40 ++
...TranslateResourceControllerIntegrationTest.java | 25 +-
.../UnpackResourceControllerIntegrationTest.java | 38 ++
.../src/test/resources/test.html | 1 +
.../src/test/resources/test.json | 1 +
.../tika-server-spring/src/test/resources/test.txt | 2 +
.../tika-server-spring/src/test/resources/test.zip | 1 +
.../src/main/resources/log4j2.xml | 32 -
.../src/test/resources/log4j2.xml | 32 -
264 files changed, 2539 insertions(+), 21774 deletions(-)
diff --git a/pom.xml b/pom.xml
index be20e234d..49d75ba63 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,6 @@
<module>tika-xmp</module>
<module>tika-langdetect</module>
<module>tika-pipes</module>
- <module>tika-server-spring</module>
<module>tika-app</module>
<module>tika-server</module>
<module>tika-integration-tests</module>
diff --git a/tika-app/pom.xml b/tika-app/pom.xml
index 09950e4d0..b089f9e1c 100644
--- a/tika-app/pom.xml
+++ b/tika-app/pom.xml
@@ -63,10 +63,9 @@
<artifactId>tika-xmp</artifactId>
<version>${project.version}</version>
</dependency>
- <!-- this brings in tika-core -->
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>tika-async-cli</artifactId>
+ <artifactId>tika-core</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
index 96276935b..b8f273575 100644
--- a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
+++ b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
@@ -65,7 +65,6 @@ import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.Tika;
-import org.apache.tika.async.cli.TikaAsyncCLI;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.config.TikaConfigSerializer;
import org.apache.tika.detect.CompositeDetector;
@@ -226,9 +225,6 @@ public class TikaCLI {
if (cli.testForHelp(args)) {
cli.usage();
return;
- } else if (cli.testForAsync(args)) {
- async(args);
- return;
}
if (args.length > 0) {
@@ -253,19 +249,6 @@ public class TikaCLI {
}
}
- private static void async(String[] args) throws Exception {
- String tikaConfigPath = "";
- String config = "--config=";
- for (String arg : args) {
- if (arg.startsWith(config)) {
- tikaConfigPath = arg.substring(config.length());
- TikaAsyncCLI.main(new String[]{tikaConfigPath});
- return;
- }
- }
- TikaAsyncCLI.main(args);
- }
-
/**
* Returns a output writer with the given encoding.
*
@@ -317,27 +300,6 @@ public class TikaCLI {
return handler;
}
- private boolean testForAsync(String[] args) {
- if (args.length == 2) {
- if (Files.isDirectory(Paths.get(args[0]))) {
- return true;
- }
- }
- for (String arg : args) {
- if (arg.equals("-a") || arg.equals("--async")) {
- return true;
- }
- if (arg.equals("-i") || arg.startsWith("--input")) {
- return true;
- }
- if (arg.equals("-o") || arg.startsWith("--output")) {
- return true;
- }
-
- }
- return false;
- }
-
public void process(String arg) throws Exception {
if (arg.equals("-?") || arg.equals("--help")) {
pipeMode = false;
@@ -544,7 +506,6 @@ public class TikaCLI {
out.println(" -J or --jsonRecursive Output metadata and content from all");
out.println(" embedded files (choose content type");
out.println(" with -x, -h, -t or -m; default is -x)");
- out.println(" -a or --async Run Tika in async mode; must specify details in a" + " tikaConfig file");
out.println(" -l or --language Output only language");
out.println(" -d or --detect Detect document type");
out.println(" --digest=X Include digest X (md2, md5, sha1,");
diff --git a/tika-bom/pom.xml b/tika-bom/pom.xml
index ff9380841..0f56c39a4 100644
--- a/tika-bom/pom.xml
+++ b/tika-bom/pom.xml
@@ -406,12 +406,6 @@
<artifactId>tika-pipes-reporter-opensearch</artifactId>
<version>4.0.0-SNAPSHOT</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-async-cli</artifactId>
- <version>4.0.0-SNAPSHOT</version>
- </dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-httpclient-commons</artifactId>
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index e3a0e9d7a..dab3cc3db 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -721,6 +721,26 @@
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java-util</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.j2objc</groupId>
+ <artifactId>j2objc-annotations</artifactId>
+ <version>3.0.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.api.grpc</groupId>
+ <artifactId>proto-google-common-protos</artifactId>
+ <version>2.51.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>failureaccess</artifactId>
+ <version>1.0.2</version>
+ </dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
diff --git a/tika-pipes/pom.xml b/tika-pipes/pom.xml
index cc06d9615..feb6c2537 100644
--- a/tika-pipes/pom.xml
+++ b/tika-pipes/pom.xml
@@ -34,6 +34,35 @@
<name>Apache Tika Pipes</name>
<description>Apache Tika Pipes aggregator project</description>
+ <properties>
+ <maven.compiler.release>17</maven.compiler.release>
+ <pf4j.version>3.12.0</pf4j.version>
+ <!-- Never include the core artifacts in your plugin lib directory. If you do, it will cause the classloading
+ to get messed up when finding your plugins. -->
+ <plugin.excluded.artifactIds>tika-core,tika-pipes-core,tika-pipe-iterators-core,tika-emitters-core,tika-fetchers-core</plugin.excluded.artifactIds>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.pf4j</groupId>
+ <artifactId>pf4j</artifactId>
+ <version>${pf4j.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons.io.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons.lang3.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<modules>
<module>tika-pipes-core</module>
<module>tika-pipes-pipe-iterators</module>
@@ -43,4 +72,4 @@
<module>tika-pipes-grpc</module>
<module>tika-pipes-cli</module>
</modules>
-</project>
\ No newline at end of file
+</project>
diff --git a/tika-pipes/tika-pipes-cli/pom.xml b/tika-pipes/tika-pipes-cli/pom.xml
index 96b0ee554..401b2dc20 100644
--- a/tika-pipes/tika-pipes-cli/pom.xml
+++ b/tika-pipes/tika-pipes-cli/pom.xml
@@ -1,33 +1,78 @@
-<?xm <parent>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-pipes</artifactId>
- <version>4.0.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>ion="1.0" encoding="UTF-8"?>
+<?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.tika</groupId>
- <artifactId>tika-pipes-parent</artifactId>
- <version>3.0.0-beta7</version>
+ <artifactId>tika-pipes</artifactId>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>tika-pipes-cli</artifactId>
<name>Tika Pipes CLI</name>
<description>Tika pipes clients for each fetcher.</description>
<properties>
- <guava.version>33.4.0-jre</guava.version>
+ <guava.version>33.5.0-jre</guava.version>
<jcommander.version>1.82</jcommander.version>
<os-maven-plugin.version>1.7.1</os-maven-plugin.version>
<jakarta.servlet-api.version>5.0.0</jakarta.servlet-api.version>
<javax.servlet-api.version>3.1.0</javax.servlet-api.version>
+
+ <guava.version>33.5.0-jre</guava.version>
+ <azure-identity.version>1.11.0</azure-identity.version>
+ <microsoft-graph.version>6.4.0</microsoft-graph.version>
+ <microsoft-kiota-serialization-json.version>1.1.1</microsoft-kiota-serialization-json.version>
+ <microsoft-kiota-authentication.version>1.0.4</microsoft-kiota-authentication.version>
+ <microsoft-kiota-abstractions.version>1.0.4</microsoft-kiota-abstractions.version>
+ <nimbus-jose-jwt.version>9.40</nimbus-jose-jwt.version>
+
</properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.microsoft.azure</groupId>
+ <artifactId>msal4j</artifactId>
+ <version>1.14.0</version>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna-platform</artifactId>
+ <version>5.13.0</version>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.annotation</groupId>
+ <artifactId>jakarta.annotation-api</artifactId>
+ <version>3.0.0</version>
+ </dependency>
+ <!-- Force consistent versions to resolve dependency convergence errors -->
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm</artifactId>
+ <version>9.8</version>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-context</artifactId>
+ <version>1.72.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jetbrains</groupId>
+ <artifactId>annotations</artifactId>
+ <version>16.0.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.api</groupId>
+ <artifactId>api-common</artifactId>
+ <version>2.49.0</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-pipes-grpc</artifactId>
- <version>${parent.version}</version>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>jakarta.servlet</groupId>
@@ -37,27 +82,27 @@
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-fetcher-s3</artifactId>
- <version>${parent.version}</version>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-fetcher-microsoft-graph</artifactId>
- <version>${parent.version}</version>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-fetcher-google-drive</artifactId>
- <version>${parent.version}</version>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-fetcher-file-system</artifactId>
- <version>${parent.version}</version>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-fetcher-http</artifactId>
- <version>${parent.version}</version>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.beust</groupId>
@@ -77,12 +122,10 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>${guava.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>${commons-lang3.version}</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
diff --git a/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/googledrive/GoogleDriveFetcherCli.java b/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/googledrive/GoogleDriveFetcherCli.java
index 563545ce8..2de659d1c 100644
--- a/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/googledrive/GoogleDriveFetcherCli.java
+++ b/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/googledrive/GoogleDriveFetcherCli.java
@@ -117,7 +117,7 @@ public class GoogleDriveFetcherCli {
}
});
- try (LineIterator lineIterator = new LineIterator(new FileReader(urlsToFetchFile))) {
+ try (LineIterator lineIterator = new LineIterator(new FileReader(urlsToFetchFile, StandardCharsets.UTF_8))) {
while (lineIterator.hasNext()) {
String nextS3Key = lineIterator.nextLine();
requestStreamObserver.onNext(FetchAndParseRequest
diff --git a/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/http/HttpFetcherCli.java b/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/http/HttpFetcherCli.java
index fd7e966cc..b6b86d5f9 100644
--- a/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/http/HttpFetcherCli.java
+++ b/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/http/HttpFetcherCli.java
@@ -3,6 +3,7 @@ package org.apache.tika.pipes.cli.http;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -103,7 +104,7 @@ public class HttpFetcherCli {
}
});
- try (LineIterator lineIterator = new LineIterator(new FileReader(urlsToFetchFile))) {
+ try (LineIterator lineIterator = new LineIterator(new FileReader(urlsToFetchFile, StandardCharsets.UTF_8))) {
while (lineIterator.hasNext()) {
String fetchKey = lineIterator.nextLine();
requestStreamObserver.onNext(FetchAndParseRequest
diff --git a/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/microsoftgraph/MicrosoftGraphFetcherCli.java b/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/microsoftgraph/MicrosoftGraphFetcherCli.java
index 6b9290eef..f622fc336 100644
--- a/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/microsoftgraph/MicrosoftGraphFetcherCli.java
+++ b/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/microsoftgraph/MicrosoftGraphFetcherCli.java
@@ -6,6 +6,7 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -130,7 +131,7 @@ public class MicrosoftGraphFetcherCli {
}
});
- try (LineIterator lineIterator = new LineIterator(new FileReader(urlsToFetchFile))) {
+ try (LineIterator lineIterator = new LineIterator(new FileReader(urlsToFetchFile, StandardCharsets.UTF_8))) {
while (lineIterator.hasNext()) {
String fetchKey = lineIterator.nextLine();
requestStreamObserver.onNext(FetchAndParseRequest
diff --git a/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/s3/S3FetcherCli.java b/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/s3/S3FetcherCli.java
index 41f298d2f..35a0f1a53 100644
--- a/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/s3/S3FetcherCli.java
+++ b/tika-pipes/tika-pipes-cli/src/main/java/org/apache/tika/pipes/cli/s3/S3FetcherCli.java
@@ -5,6 +5,7 @@ import static org.apache.tika.pipes.cli.mapper.ObjectMapperProvider.OBJECT_MAPPE
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -204,7 +205,7 @@ public class S3FetcherCli {
}
});
- try (LineIterator lineIterator = new LineIterator(new FileReader(urlsToFetchFile))) {
+ try (LineIterator lineIterator = new LineIterator(new FileReader(urlsToFetchFile, StandardCharsets.UTF_8))) {
while (lineIterator.hasNext()) {
String nextS3Key = lineIterator.nextLine();
requestStreamObserver.onNext(FetchAndParseRequest
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/HandlerConfig.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/HandlerConfig.java
new file mode 100644
index 000000000..fd93dc530
--- /dev/null
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/HandlerConfig.java
@@ -0,0 +1,192 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ *
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.core;
+
+import java.io.Serializable;
+import java.util.Locale;
+import java.util.Objects;
+
+import org.apache.tika.sax.BasicContentHandlerFactory;
+
+public class HandlerConfig implements Serializable {
+
+ /**
+ * Serial version UID
+ */
+ private static final long serialVersionUID = -3861669115439125268L;
+
+ public static final HandlerConfig DEFAULT_HANDLER_CONFIG =
+ new HandlerConfig(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, PARSE_MODE.RMETA,
+ -1, -1, true);
+
+ /**
+ * {@link PARSE_MODE#RMETA} "recursive metadata" is the same as the -J option
+ * in tika-app and the /rmeta endpoint in tika-server. Each embedded file is represented as
+ * its own metadata object.
+ *
+ * {@link PARSE_MODE#CONCATENATE} is similar
+ * to the legacy tika-app behavior and the /tika endpoint (accept: application/json) in
+ * tika-server. This concatenates the
+ * contents of embedded files and returns a single metadata object for the file no
+ * matter how many embedded objects there are; this option throws away metadata from
+ * embedded objects and silently skips exceptions in embedded objects.
+ */
+ public enum PARSE_MODE {
+ RMETA,
+ CONCATENATE;
+
+ public static PARSE_MODE parseMode(String modeString) {
+ for (PARSE_MODE m : PARSE_MODE.values()) {
+ if (m.name().equalsIgnoreCase(modeString)) {
+ return m;
+ }
+ }
+ StringBuilder sb = new StringBuilder();
+ int i = 0;
+ for (PARSE_MODE m : PARSE_MODE.values()) {
+ if (i++ > 0) {
+ sb.append(", ");
+ }
+ sb.append(m.name().toLowerCase(Locale.US));
+ }
+ throw new IllegalArgumentException("mode must be one of: (" + sb +
+ "). I regret I do not understand: " + modeString);
+ }
+ }
+
+ private BasicContentHandlerFactory.HANDLER_TYPE type =
+ BasicContentHandlerFactory.HANDLER_TYPE.TEXT;
+
+ int writeLimit = -1;
+ int maxEmbeddedResources = -1;
+
+ boolean throwOnWriteLimitReached = true;
+ PARSE_MODE parseMode = PARSE_MODE.RMETA;
+
+
+ public HandlerConfig() {
+
+ }
+
+ public HandlerConfig(BasicContentHandlerFactory.HANDLER_TYPE type, PARSE_MODE parseMode,
+ int writeLimit,
+ int maxEmbeddedResources, boolean throwOnWriteLimitReached) {
+ this.type = type;
+ this.parseMode = parseMode;
+ this.writeLimit = writeLimit;
+ this.maxEmbeddedResources = maxEmbeddedResources;
+ this.throwOnWriteLimitReached = throwOnWriteLimitReached;
+ }
+
+ public BasicContentHandlerFactory.HANDLER_TYPE getType() {
+ return type;
+ }
+
+ public void setType(BasicContentHandlerFactory.HANDLER_TYPE type) {
+ this.type = type;
+ }
+
+ public void setType(String typeString) {
+ setType(BasicContentHandlerFactory.HANDLER_TYPE.valueOf(typeString));
+ }
+
+ public int getWriteLimit() {
+ return writeLimit;
+ }
+
+ public void setWriteLimit(int writeLimit) {
+ this.writeLimit = writeLimit;
+ }
+
+ public int getMaxEmbeddedResources() {
+ return maxEmbeddedResources;
+ }
+
+ public void setMaxEmbeddedResources(int maxEmbeddedResources) {
+ this.maxEmbeddedResources = maxEmbeddedResources;
+ }
+
+ public boolean isThrowOnWriteLimitReached() {
+ return throwOnWriteLimitReached;
+ }
+
+ public void setThrowOnWriteLimitReached(boolean throwOnWriteLimitReached) {
+ this.throwOnWriteLimitReached = throwOnWriteLimitReached;
+ }
+
+ public PARSE_MODE getParseMode() {
+ return parseMode;
+ }
+
+ public void setParseMode(PARSE_MODE parseMode) {
+ this.parseMode = parseMode;
+ }
+
+ public void setParseMode(String parseMode) {
+ this.parseMode = PARSE_MODE.parseMode(parseMode);
+ }
+
+ @Override
+ public String toString() {
+ return "HandlerConfig{" + "type=" + type + ", writeLimit=" + writeLimit + ", maxEmbeddedResources=" + maxEmbeddedResources +
+ ", throwOnWriteLimitReached=" + throwOnWriteLimitReached + ", parseMode=" + parseMode + '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ HandlerConfig that = (HandlerConfig) o;
+ return writeLimit == that.writeLimit && maxEmbeddedResources == that.maxEmbeddedResources && throwOnWriteLimitReached == that.throwOnWriteLimitReached &&
+ type == that.type && parseMode == that.parseMode;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Objects.hashCode(type);
+ result = 31 * result + writeLimit;
+ result = 31 * result + maxEmbeddedResources;
+ result = 31 * result + Boolean.hashCode(throwOnWriteLimitReached);
+ result = 31 * result + Objects.hashCode(parseMode);
+ return result;
+ }
+}
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/parser/ParseService.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/parser/ParseService.java
index a0d25719f..0efe66cc9 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/parser/ParseService.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/parser/ParseService.java
@@ -24,7 +24,7 @@ import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.parser.ocr.TesseractOCRConfig;
-import org.apache.tika.pipes.HandlerConfig;
+import org.apache.tika.pipes.core.HandlerConfig;
import org.apache.tika.pipes.core.exception.TikaServerParseException;
import org.apache.tika.sax.BasicContentHandlerFactory;
import org.apache.tika.sax.RecursiveParserWrapperHandler;
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/util/OSUtils.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/util/OSUtils.java
deleted file mode 100644
index 892e5367d..000000000
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/util/OSUtils.java
+++ /dev/null
@@ -1,258 +0,0 @@
-package org.apache.tika.pipes.core.util;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.google.common.collect.Lists;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class OSUtils {
- private static final Logger LOG = LoggerFactory.getLogger(OSUtils.class);
-
- private static String OS = System.getProperty("os.name").toLowerCase();
-
- private static final String KILL_WINDOWS_TASK = "taskkill /F /PID %d";
- private static final String KILL_NIX_TASK = "kill -s TERM %d";
-
- public static boolean isWindows() {
- return (OS.indexOf("win") >= 0);
- }
-
- public static boolean isMac() {
- return (OS.indexOf("mac") >= 0);
- }
-
- /**
- * Gets command line for a given process.
- *
- * @param pid The PID of the process.
- * @return Command line for the process if it exists.
- */
- /**
- * Get process id's for a certain executable regardless of operating system.
- *
- * @param executablePathContains The path of the executable you want to get PID's of.
- * @return Set of command lines
- */
- public static List<String> getCommandLinesFor(String executablePathContains) {
- List<String> commandLines = Lists.newArrayList();
- if (isWindows()) {
- try {
- String line;
- String processCmd = System.getenv("windir") + "\\system32\\Wbem\\wmic process get commandline,processid";
- Process p = Runtime.getRuntime().exec(processCmd);
- LOG.debug("Getting windows process id's for {} processes with cmd \"{}\"", executablePathContains, processCmd);
- try (BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
- while ((line = input.readLine()) != null) {
- LOG.debug("wmic process line: {}", line);
- if (line.toLowerCase().contains(executablePathContains.toLowerCase())) {
- commandLines.add(line);
- }
- }
- }
- } catch (Exception e) {
- LOG.error("Couldn't get process list on windows", e);
- }
- } else if (isMac()) {
- try {
- String line;
- String processCmd = "ps aux";
- Process p = Runtime.getRuntime().exec(processCmd);
- LOG.debug("Getting mac process id's for processes command line containing {} with cmd \"{}\"",
- executablePathContains, processCmd);
- try (BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
- while ((line = input.readLine()) != null) {
- LOG.debug("ps aux line: {}", line);
- if (line.contains(executablePathContains)) {
- commandLines.add(line);
- }
- }
- }
- } catch (Exception e) {
- LOG.error("Couldn't get process list", e);
- }
- } else {
- try {
- String line;
- String processCmd = String.format("ps -eo pid,args");
- LOG.debug("Getting linux process id's where command line contains {} with cmd \"{}\"",
- executablePathContains, processCmd);
- Process p = Runtime.getRuntime().exec(processCmd);
- try (BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
- while ((line = input.readLine()) != null) {
- LOG.debug("ps line: {}", line);
- if (line.contains(executablePathContains)) {
- commandLines.add(line);
- }
- }
- }
- } catch (Exception e) {
- LOG.error("Couldn't get process list", e);
- }
- }
- return commandLines;
- }
-
- /**
- * Get java process id's for a certain executable regardless of operating system.
- *
- * @param executablePathContains The path of the executable you want to get PID's of.
- * @param parentPid The parent PID of the parent.
- * @return Set of the PID's.
- */
- public static Set<Integer> getJavaProcessIds(String executablePathContains, Integer parentPid) {
- Set<Integer> processIds = new HashSet<>();
- if (isWindows()) {
- try {
- String line;
- String processCmd = System.getenv("windir") + "\\system32\\Wbem\\wmic process where " +
- "\"name like '%java%'\" get processid,commandline";
- Process p = Runtime.getRuntime().exec(processCmd);
- LOG.debug("Getting windows process id's for {} processes with cmd \"{}\"", executablePathContains, processCmd);
- try (BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
- while ((line = input.readLine()) != null) {
- LOG.debug("wmic process line: {}", line);
- if (line.toLowerCase().contains(executablePathContains.toLowerCase())) {
- if (parentPid == null || line.toLowerCase().contains(parentPid + "")) {
- String[] splitLine = line.trim().split("\\s+");
- int pid = Integer.parseInt(splitLine[splitLine.length - 1]);
- processIds.add(pid);
- LOG.info("Process found: pid={}, commandLine={}", pid, line);
- }
- }
- }
- }
- } catch (Exception e) {
- LOG.error("Couldn't get process list on windows", e);
- }
- } else if (isMac()) {
- try {
- String line;
- String processCmd = "ps aux";
- Process p = Runtime.getRuntime().exec(processCmd);
- LOG.debug("Getting mac process id's for processes command line containing {} with cmd \"{}\"",
- executablePathContains, processCmd);
- try (BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
- while ((line = input.readLine()) != null) {
- LOG.debug("ps aux line: {}", line);
- if (line.contains(executablePathContains)) {
- if (parentPid == null || line.toLowerCase().contains(parentPid + "")) {
- String[] splitLine = line.trim().split("\\s+");
- int pid = Integer.parseInt(splitLine[1]);
- processIds.add(pid);
- LOG.info("Process found: pid={}, commandLine={}", pid, line);
- }
- }
- }
- }
- } catch (Exception e) {
- LOG.error("Couldn't get process list", e);
- }
- } else {
- try {
- String line;
- String processCmd = String.format("ps -eo pid,args");
- LOG.debug("Getting linux process id's where command line contains {} with cmd \"{}\"",
- executablePathContains, processCmd);
- Process p = Runtime.getRuntime().exec(processCmd);
- try (BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
- while ((line = input.readLine()) != null) {
- LOG.debug("ps line: {}", line);
- if (line.contains(executablePathContains)) {
- if (parentPid == null || line.toLowerCase().contains(parentPid + "")) {
- String[] splitLine = line.trim().split("\\s+");
- int pid = Integer.parseInt(splitLine[0]);
- processIds.add(pid);
- LOG.info("Process found: pid={}, commandLine={}", pid, line);
- }
- }
- }
- }
- } catch (Exception e) {
- LOG.error("Couldn't get process list", e);
- }
- }
- return processIds;
- }
-
- /**
- * Get process id's for a certain executable regardless of operating system.
- *
- * @param executablePathContains The path of the executable you want to get PID's of.
- * @return Set of the PID's.
- */
- public static Set<Integer> getJavaProcessIds(String executablePathContains) {
- return getJavaProcessIds(executablePathContains, null);
- }
-
- /**
- * Kill the process with the quit signal.
- *
- * @param pid Process ID to kill.
- */
- public static void forceKillPid(int pid) {
- if (isWindows()) {
- try {
- String cmd = String.format(KILL_WINDOWS_TASK, pid);
- LOG.info("Killing process with command: {}", cmd);
- Runtime.getRuntime().exec(cmd);
- } catch (IOException e) {
- LOG.error("Couldn't kill process {}", pid, e);
- }
- } else {
- try {
- String cmd = String.format(KILL_NIX_TASK, pid);
- LOG.info("Killing process with command: {}", cmd);
- Runtime.getRuntime().exec(cmd);
- } catch (IOException e) {
- LOG.error("Couldn't kill process {}", pid, e);
- }
- }
- }
-
- /**
- * Checks whether the supplied port is available on any local address.
- *
- * @param port the port to check for.
- * @return <code>true</code> if the port is available, otherwise <code>false</code>.
- */
- public static boolean isPortAvailable(int port) {
- ServerSocket socket;
- try {
- socket = new ServerSocket();
- } catch (IOException e) {
- throw new IllegalStateException("Unable to create ServerSocket.", e);
- }
-
- try {
- InetSocketAddress sa = new InetSocketAddress(port);
- socket.bind(sa);
- return true;
- } catch (IOException ex) {
- return false;
- } finally {
- try {
- socket.close();
- } catch (IOException ex) {
- }
- }
- }
-
- public static int findAvailablePort(int minPort, int maxPort) {
- int candidatePort = minPort;
- while (!isPortAvailable(candidatePort)) {
- ++candidatePort;
- if (candidatePort > maxPort) {
- throw new IllegalStateException(String.format("Could not find an available port in the range [%d, %d]", minPort, maxPort));
- }
- }
- return candidatePort;
- }
-}
diff --git a/tika-pipes/tika-pipes-emitters/tika-emitter-file-system/pom.xml b/tika-pipes/tika-pipes-emitters/tika-emitter-file-system/pom.xml
index 4151ddf0c..822a7e66f 100644
--- a/tika-pipes/tika-pipes-emitters/tika-emitter-file-system/pom.xml
+++ b/tika-pipes/tika-pipes-emitters/tika-emitter-file-system/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.tika</groupId>
<artifactId>tika-pipes-emitters</artifactId>
- <version>3.0.0-beta7</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>tika-emitter-file-system</artifactId>
diff --git a/tika-pipes/tika-pipes-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/fetchers/filesystem/FileSystemEmitter.java b/tika-pipes/tika-pipes-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/fetchers/filesystem/FileSystemEmitter.java
index 329f590d9..2d2020f0a 100644
--- a/tika-pipes/tika-pipes-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/fetchers/filesystem/FileSystemEmitter.java
+++ b/tika-pipes/tika-pipes-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/fetchers/filesystem/FileSystemEmitter.java
@@ -24,6 +24,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
+import java.util.Locale;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.FilenameUtils;
@@ -56,7 +57,7 @@ public class FileSystemEmitter implements Emitter {
String addFileExtension = fileSystemEmitterConfig.getAddFileExtension();
OnExistBehavior onExists = OnExistBehavior.valueOf(fileSystemEmitterConfig
.getOnExists()
- .toUpperCase());
+ .toUpperCase(Locale.ROOT));
for (EmitOutput emitOutput : emitOutputs) {
Path output;
String emitKey = FilenameUtils.getName(emitOutput.getFetchKey());
diff --git a/tika-pipes/tika-pipes-emitters/tika-emitter-file-system/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-emitters/tika-emitter-file-system/src/main/resources/plugin.properties
index a2c9fb533..6d5bf16f9 100644
--- a/tika-pipes/tika-pipes-emitters/tika-emitter-file-system/src/main/resources/plugin.properties
+++ b/tika-pipes/tika-pipes-emitters/tika-emitter-file-system/src/main/resources/plugin.properties
@@ -16,6 +16,6 @@
plugin.id=filesystem-emitter
plugin.class=org.apache.tika.pipes.fetchers.filesystem.FileSystemEmitterPlugin
-plugin.version=3.0.0-beta7
+plugin.version=4.0.0-SNAPSHOT
plugin.provider=File System Emitter
plugin.description=File system Emitter
diff --git a/tika-pipes/tika-pipes-emitters/tika-emitters-core/pom.xml b/tika-pipes/tika-pipes-emitters/tika-emitters-core/pom.xml
index 0a2cf9637..2df016167 100644
--- a/tika-pipes/tika-pipes-emitters/tika-emitters-core/pom.xml
+++ b/tika-pipes/tika-pipes-emitters/tika-emitters-core/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.tika</groupId>
<artifactId>tika-pipes-emitters</artifactId>
- <version>3.0.0-beta7</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>tika-emitters-core</artifactId>
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetcher-atlassian-jwt/pom.xml b/tika-pipes/tika-pipes-fetchers/tika-fetcher-atlassian-jwt/pom.xml
index 12fbb2689..97e1707db 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetcher-atlassian-jwt/pom.xml
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetcher-atlassian-jwt/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.tika</groupId>
<artifactId>tika-pipes-fetchers</artifactId>
- <version>3.0.0-beta7</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>tika-fetcher-atlassian-jwt</artifactId>
@@ -13,7 +13,7 @@
<description>A Fetcher that uses Atlassian JWT authentication to acquire documents for Tika to parse.</description>
<properties>
<maven.compiler.release>17</maven.compiler.release>
- <guava.version>33.4.0-jre</guava.version>
+ <guava.version>33.5.0-jre</guava.version>
<nimbus-jose-jwt.version>9.40</nimbus-jose-jwt.version>
</properties>
<dependencies>
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetcher-atlassian-jwt/src/main/java/org/apache/tika/pipes/fetchers/atlassianjwt/AtlassianJwtGenerator.java b/tika-pipes/tika-pipes-fetchers/tika-fetcher-atlassian-jwt/src/main/java/org/apache/tika/pipes/fetchers/atlassianjwt/AtlassianJwtGenerator.java
index 75c51bee5..457f6db95 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetcher-atlassian-jwt/src/main/java/org/apache/tika/pipes/fetchers/atlassianjwt/AtlassianJwtGenerator.java
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetcher-atlassian-jwt/src/main/java/org/apache/tika/pipes/fetchers/atlassianjwt/AtlassianJwtGenerator.java
@@ -29,6 +29,7 @@ import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.stream.Collectors;
@@ -129,7 +130,7 @@ public class AtlassianJwtGenerator {
}
String canonicalQueryString = generateCanonicalQueryString(url.getQuery());
- String canonicalRequest = httpMethod.toUpperCase() + "&" + path + "&" + canonicalQueryString;
+ String canonicalRequest = httpMethod.toUpperCase(Locale.ROOT) + "&" + path + "&" + canonicalQueryString;
return canonicalRequest;
}
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetcher-atlassian-jwt/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-fetchers/tika-fetcher-atlassian-jwt/src/main/resources/plugin.properties
index 23ea2d0e2..eb5c74068 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetcher-atlassian-jwt/src/main/resources/plugin.properties
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetcher-atlassian-jwt/src/main/resources/plugin.properties
@@ -1,6 +1,6 @@
plugin.id=atlassian-jwt-fetcher
plugin.class=org.apache.tika.pipes.fetchers.atlassianjwt.AtlassianJwtFetcherPlugin
-plugin.version=3.0.0-beta7
+plugin.version=4.0.0-SNAPSHOT
plugin.provider=Apache Tika
plugin.dependencies=
plugin.description=A Fetcher that uses Atlassian JWT authentication to acquire documents for Tika to parse.
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetcher-file-system/pom.xml b/tika-pipes/tika-pipes-fetchers/tika-fetcher-file-system/pom.xml
index d714767b0..47b3072f5 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetcher-file-system/pom.xml
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetcher-file-system/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.tika</groupId>
<artifactId>tika-pipes-fetchers</artifactId>
- <version>3.0.0-beta7</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>tika-fetcher-file-system</artifactId>
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetcher-file-system/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-fetchers/tika-fetcher-file-system/src/main/resources/plugin.properties
index 26a59a211..157817624 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetcher-file-system/src/main/resources/plugin.properties
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetcher-file-system/src/main/resources/plugin.properties
@@ -16,6 +16,6 @@
plugin.id=filesystem-fetcher
plugin.class=org.apache.tika.pipes.fetchers.filesystem.FileSystemFetcherPlugin
-plugin.version=3.0.0-beta7
+plugin.version=4.0.0-SNAPSHOT
plugin.provider=File System Fetcher
plugin.description=File system fetcher
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetcher-google-drive/pom.xml b/tika-pipes/tika-pipes-fetchers/tika-fetcher-google-drive/pom.xml
index 17987d4c5..fe17394c0 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetcher-google-drive/pom.xml
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetcher-google-drive/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.tika</groupId>
<artifactId>tika-pipes-fetchers</artifactId>
- <version>3.0.0-beta7</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>tika-fetcher-google-drive</artifactId>
@@ -13,27 +13,78 @@
<description>A Fetcher that uses an Google Drive to acquire documents for Tika to parse.</description>
<properties>
<maven.compiler.release>17</maven.compiler.release>
- <guava.version>33.4.0-jre</guava.version>
+ <guava.version>33.5.0-jre</guava.version>
<azure-identity.version>1.11.0</azure-identity.version>
<microsoft-graph.version>6.4.0</microsoft-graph.version>
<microsoft-kiota-serialization-json.version>1.1.1</microsoft-kiota-serialization-json.version>
<nimbus-jose-jwt.version>9.40</nimbus-jose-jwt.version>
<aws.version>1.12.744</aws.version>
<google-api-services-drive.version>v3-rev20241027-2.0.0</google-api-services-drive.version>
- <google-auth-library-oauth2-http.version>1.30.0</google-auth-library-oauth2-http.version>
- <google-api-client.version>1.33.0</google-api-client.version>
+ <google-api-client.version>2.8.1</google-api-client.version>
+ <google-http-client.version>2.0.0</google-http-client.version>
</properties>
<dependencies>
<dependency>
<groupId>com.google.api-client</groupId>
<artifactId>google-api-client</artifactId>
<version>${google-api-client.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.http-client</groupId>
+ <artifactId>google-http-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.http-client</groupId>
+ <artifactId>google-http-client-gson</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.http-client</groupId>
+ <artifactId>google-http-client-apache-v2</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
+ <!-- Force consistent HTTP client versions -->
<dependency>
- <groupId>com.google.auth</groupId>
- <artifactId>google-auth-library-oauth2-http</artifactId>
- <version>${google-auth-library-oauth2-http.version}</version>
+ <groupId>com.google.http-client</groupId>
+ <artifactId>google-http-client</artifactId>
+ <version>${google-http-client.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.opencensus</groupId>
+ <artifactId>opencensus-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.google.http-client</groupId>
+ <artifactId>google-http-client-gson</artifactId>
+ <version>${google-http-client.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.http-client</groupId>
+ <artifactId>google-http-client-apache-v2</artifactId>
+ <version>${google-http-client.version}</version>
+ </dependency>
+
+ <!-- Force consistent grpc-context version -->
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-context</artifactId>
+ <version>1.70.0</version>
+ </dependency>
+
+ <!-- Add opencensus-api without grpc-context conflict -->
+ <dependency>
+ <groupId>io.opencensus</groupId>
+ <artifactId>opencensus-api</artifactId>
+ <version>0.31.1</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-context</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- Google Drive API -->
@@ -41,6 +92,12 @@
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-drive</artifactId>
<version>${google-api-services-drive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.api-client</groupId>
+ <artifactId>google-api-client</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.pf4j</groupId>
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetcher-google-drive/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-fetchers/tika-fetcher-google-drive/src/main/resources/plugin.properties
index 1caec187b..b5a173779 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetcher-google-drive/src/main/resources/plugin.properties
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetcher-google-drive/src/main/resources/plugin.properties
@@ -16,6 +16,6 @@
plugin.id=google-drive-fetcher
plugin.class=org.apache.tika.pipes.fetchers.googledrive.GoogleDrivePlugin
-plugin.version=3.0.0-beta7
+plugin.version=4.0.0-SNAPSHOT
plugin.provider=Google Drive Fetcher
plugin.description=Fetcher capable of obtaining binary content from Google Drive
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetcher-http/pom.xml b/tika-pipes/tika-pipes-fetchers/tika-fetcher-http/pom.xml
index d0e283e22..7e7856cc7 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetcher-http/pom.xml
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetcher-http/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.tika</groupId>
<artifactId>tika-pipes-fetchers</artifactId>
- <version>3.0.0-beta7</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>tika-fetcher-http</artifactId>
@@ -13,7 +13,7 @@
<description>A Fetcher that uses an HTTP client to acquire documents for Tika to parse.</description>
<properties>
<maven.compiler.release>17</maven.compiler.release>
- <guava.version>33.4.0-jre</guava.version>
+ <guava.version>33.5.0-jre</guava.version>
<nimbus-jose-jwt.version>9.40</nimbus-jose-jwt.version>
</properties>
<dependencies>
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetcher-http/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-fetchers/tika-fetcher-http/src/main/resources/plugin.properties
index 23c3af057..1c7ba9800 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetcher-http/src/main/resources/plugin.properties
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetcher-http/src/main/resources/plugin.properties
@@ -16,6 +16,6 @@
plugin.id=http-fetcher
plugin.class=org.apache.tika.pipes.fetchers.http.HttpFetcherPlugin
-plugin.version=3.0.0-beta7
+plugin.version=4.0.0-SNAPSHOT
plugin.provider=HTTP Fetcher
plugin.description=HTTP fetcher
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetcher-microsoft-graph/pom.xml b/tika-pipes/tika-pipes-fetchers/tika-fetcher-microsoft-graph/pom.xml
index a7ea0ce72..24c5119c2 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetcher-microsoft-graph/pom.xml
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetcher-microsoft-graph/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.tika</groupId>
<artifactId>tika-pipes-fetchers</artifactId>
- <version>3.0.0-beta7</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>tika-fetcher-microsoft-graph</artifactId>
@@ -13,7 +13,7 @@
<description>A Fetcher that uses an Microsoft Graph Fetcher to acquire documents for Tika to parse.</description>
<properties>
<maven.compiler.release>17</maven.compiler.release>
- <guava.version>33.4.0-jre</guava.version>
+ <guava.version>33.5.0-jre</guava.version>
<azure-identity.version>1.11.0</azure-identity.version>
<microsoft-graph.version>6.4.0</microsoft-graph.version>
<microsoft-kiota-serialization-json.version>1.1.1</microsoft-kiota-serialization-json.version>
@@ -21,6 +21,27 @@
<microsoft-kiota-abstractions.version>1.0.4</microsoft-kiota-abstractions.version>
<nimbus-jose-jwt.version>9.40</nimbus-jose-jwt.version>
</properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.microsoft.azure</groupId>
+ <artifactId>msal4j</artifactId>
+ <version>1.14.0</version>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna-platform</artifactId>
+ <version>5.13.0</version>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.annotation</groupId>
+ <artifactId>jakarta.annotation-api</artifactId>
+ <version>3.0.0</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<dependency>
<groupId>io.opentelemetry</groupId>
@@ -34,12 +55,6 @@
<version>1.43.0</version>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>jakarta.annotation</groupId>
- <artifactId>jakarta.annotation-api</artifactId>
- <version>2.1.1</version>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>io.github.std-uritemplate</groupId>
<artifactId>std-uritemplate</artifactId>
@@ -169,7 +184,6 @@
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<includeScope>compile</includeScope>
- <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds>
</configuration>
</execution>
</executions>
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetcher-microsoft-graph/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-fetchers/tika-fetcher-microsoft-graph/src/main/resources/plugin.properties
index 7d8358195..e6a98f750 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetcher-microsoft-graph/src/main/resources/plugin.properties
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetcher-microsoft-graph/src/main/resources/plugin.properties
@@ -16,6 +16,6 @@
plugin.id=microsoft-graph-fetcher
plugin.class=org.apache.tika.pipes.fetchers.microsoftgraph.MicrosoftGraphPlugin
-plugin.version=3.0.0-beta7
+plugin.version=4.0.0-SNAPSHOT
plugin.provider=Microsoft Graph Fetcher
plugin.description=Microsoft Graph fetcher
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetcher-s3/pom.xml b/tika-pipes/tika-pipes-fetchers/tika-fetcher-s3/pom.xml
index 1a0e4547f..21f334055 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetcher-s3/pom.xml
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetcher-s3/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.tika</groupId>
<artifactId>tika-pipes-fetchers</artifactId>
- <version>3.0.0-beta7</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>tika-fetcher-s3</artifactId>
@@ -13,9 +13,26 @@
<description>A Fetcher that uses an S3 to acquire documents for Tika to parse.</description>
<properties>
<maven.compiler.release>17</maven.compiler.release>
- <guava.version>33.4.0-jre</guava.version>
+ <guava.version>33.5.0-jre</guava.version>
<s3.version>2.30.4</s3.version>
</properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <!-- Force consistent versions to resolve AWS SDK convergence issues -->
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>netty-nio-client</artifactId>
+ <version>${s3.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>apache-client</artifactId>
+ <version>${s3.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetcher-s3/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-fetchers/tika-fetcher-s3/src/main/resources/plugin.properties
index 6961e2ce1..87666c740 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetcher-s3/src/main/resources/plugin.properties
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetcher-s3/src/main/resources/plugin.properties
@@ -16,6 +16,6 @@
plugin.id=s3-fetcher
plugin.class=org.apache.tika.pipes.fetchers.s3.S3Plugin
-plugin.version=3.0.0-beta7
+plugin.version=4.0.0-SNAPSHOT
plugin.provider=S3 Fetcher
plugin.description=Fetcher capable of obtaining binary content from Amazon S3 SDK
diff --git a/tika-pipes/tika-pipes-fetchers/tika-fetchers-core/pom.xml b/tika-pipes/tika-pipes-fetchers/tika-fetchers-core/pom.xml
index 6a6afc105..2fc6a303c 100644
--- a/tika-pipes/tika-pipes-fetchers/tika-fetchers-core/pom.xml
+++ b/tika-pipes/tika-pipes-fetchers/tika-fetchers-core/pom.xml
@@ -12,7 +12,38 @@
<name>Tika Pipes Fetchers Core Library</name>
<properties>
<maven.compiler.release>17</maven.compiler.release>
+ <protobuf.version>3.25.8</protobuf.version>
+ <j2objc-annotations.version>3.0.0</j2objc-annotations.version>
+ <grpc-google.version>2.51.0</grpc-google.version>
+ <failureaccess.version>1.0.2</failureaccess.version>
</properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <!-- Force consistent versions to resolve convergence issues -->
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java-util</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.j2objc</groupId>
+ <artifactId>j2objc-annotations</artifactId>
+ <version>${j2objc-annotations.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.api.grpc</groupId>
+ <artifactId>proto-google-common-protos</artifactId>
+ <version>${grpc-google.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>failureaccess</artifactId>
+ <version>${failureaccess.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
diff --git a/tika-pipes/tika-pipes-grpc/docker-build/Dockerfile b/tika-pipes/tika-pipes-grpc/docker-build/Dockerfile
index 8008f2f23..4867b5d90 100644
--- a/tika-pipes/tika-pipes-grpc/docker-build/Dockerfile
+++ b/tika-pipes/tika-pipes-grpc/docker-build/Dockerfile
@@ -4,7 +4,7 @@ COPY plugins/ /tika/plugins/
COPY config/ /tika/config/
COPY bin/ /tika/bin
ARG JRE='openjdk-17-jre-headless'
-ARG VERSION='3.0.0-beta7'
+ARG VERSION='4.0.0-SNAPSHOT'
ARG TIKA_PIPES_MAX_INBOUND_MESSAGE_SIZE=104857600
ARG TIKA_PIPES_MAX_OUTBOUND_MESSAGE_SIZE=104857600
ARG TIKA_PIPES_GRPC_NUM_THREADS=4
diff --git a/tika-pipes/tika-pipes-grpc/pom.xml b/tika-pipes/tika-pipes-grpc/pom.xml
index 688d3cebf..d2d561af5 100644
--- a/tika-pipes/tika-pipes-grpc/pom.xml
+++ b/tika-pipes/tika-pipes-grpc/pom.xml
@@ -15,23 +15,21 @@
</description>
<properties>
<maven.compiler.release>17</maven.compiler.release>
- <grpc.version>1.70.0</grpc.version>
- <protobuf.version>3.25.5</protobuf.version>
- <protoc.version>3.25.5</protoc.version>
+ <grpc.version>1.72.0</grpc.version>
+ <protobuf.version>3.25.8</protobuf.version>
+ <protoc.version>3.25.8</protoc.version>
<asarkar-grpc-test.version>1.2.2</asarkar-grpc-test.version>
<awaitility.version>4.2.1</awaitility.version>
<j2objc-annotations.version>3.0.0</j2objc-annotations.version>
- <error_prone_annotations.version>2.28.0</error_prone_annotations.version>
- <jackson-module-jsonSchema.version>2.15.0</jackson-module-jsonSchema.version>
- <guava.version>33.4.0-jre</guava.version>
+ <error_prone_annotations.version>2.42.0</error_prone_annotations.version>
+ <jackson-module-jsonSchema.version>2.18.2</jackson-module-jsonSchema.version>
+ <guava.version>33.5.0-jre</guava.version>
<spring-data-jpa.version>3.3.4</spring-data-jpa.version>
<ignite.version>2.16.0</ignite.version>
<ignite-spring-data-ext.version>3.0.0</ignite-spring-data-ext.version>
<jakarta.persistence-api.version>3.2.0</jakarta.persistence-api.version>
- <tika-version>3.0.0</tika-version>
- <commons-io.version>2.18.0</commons-io.version>
<json-schema-validator.version>2.2.14</json-schema-validator.version>
- <grpc-starter-dependencies.version>3.5.0.1</grpc-starter-dependencies.version>
+ <grpc-starter-dependencies.version>3.5.5</grpc-starter-dependencies.version>
</properties>
<dependencyManagement>
<dependencies>
@@ -60,17 +58,17 @@
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-parsers-standard-package</artifactId>
- <version>${tika-version}</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
- <version>${tika-version}</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-parser-microsoft-module</artifactId>
- <version>${tika-version}</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
@@ -84,11 +82,6 @@
<version>${json-schema-validator.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>${commons-io.version}</version>
- </dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
@@ -141,7 +134,6 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>3.25.4</version>
</dependency>
<dependency>
<groupId>org.hsqldb</groupId>
@@ -278,11 +270,6 @@
<artifactId>jakarta.persistence-api</artifactId>
<version>${jakarta.persistence-api.version}</version>
</dependency>
- <dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
- <version>1.4.197</version>
- </dependency>
<dependency>
<groupId>org.pf4j</groupId>
<artifactId>pf4j</artifactId>
@@ -377,7 +364,7 @@
<argument>${project.basedir}/docker-build/docker-build.sh</argument>
</arguments>
<environmentVariables>
- <TIKA_PIPES_VERSION>${version}</TIKA_PIPES_VERSION>
+ <TIKA_PIPES_VERSION>${project.version}</TIKA_PIPES_VERSION>
</environmentVariables>
</configuration>
</execution>
diff --git a/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerFetchersCrudTest.java b/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerFetchersCrudTest.java
index e492aa85d..d0e11a588 100644
--- a/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerFetchersCrudTest.java
+++ b/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerFetchersCrudTest.java
@@ -20,7 +20,7 @@ import org.apache.tika.SaveFetcherReply;
import org.apache.tika.SaveFetcherRequest;
import org.apache.tika.TikaGrpc;
import org.apache.tika.pipes.TikaPipesIntegrationTestBase;
-import org.apache.tika.pipes.fetcher.fs.config.FileSystemFetcherConfig;
+import org.apache.tika.pipes.fetchers.filesystem.FileSystemFetcherConfig;
class TikaGrpcServerFetchersCrudTest extends TikaPipesIntegrationTestBase {
@Test
diff --git a/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerJsonSchemaTest.java b/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerJsonSchemaTest.java
index 64d1d68d2..5ff8fe5eb 100644
--- a/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerJsonSchemaTest.java
+++ b/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerJsonSchemaTest.java
@@ -24,7 +24,7 @@ import org.apache.tika.SaveFetcherReply;
import org.apache.tika.SaveFetcherRequest;
import org.apache.tika.TikaGrpc;
import org.apache.tika.pipes.TikaPipesIntegrationTestBase;
-import org.apache.tika.pipes.fetcher.fs.config.FileSystemFetcherConfig;
+import org.apache.tika.pipes.fetchers.filesystem.FileSystemFetcherConfig;
class TikaGrpcServerJsonSchemaTest extends TikaPipesIntegrationTestBase {
@Test
diff --git a/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerPipeJobTest.java b/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerPipeJobTest.java
index 1d19af0be..bdd05360b 100644
--- a/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerPipeJobTest.java
+++ b/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerPipeJobTest.java
@@ -42,9 +42,9 @@ import org.apache.tika.SavePipeIteratorReply;
import org.apache.tika.SavePipeIteratorRequest;
import org.apache.tika.TikaGrpc;
import org.apache.tika.pipes.TikaPipesIntegrationTestBase;
-import org.apache.tika.pipes.fetcher.fs.config.FileSystemFetcherConfig;
import org.apache.tika.pipes.fetchers.filesystem.CsvPipeIteratorConfig;
import org.apache.tika.pipes.fetchers.filesystem.FileSystemEmitterConfig;
+import org.apache.tika.pipes.fetchers.filesystem.FileSystemFetcherConfig;
@Slf4j
class TikaGrpcServerPipeJobTest extends TikaPipesIntegrationTestBase {
diff --git a/tika-pipes/tika-pipes-pipe-iterators/tika-pipe-iterator-csv/pom.xml b/tika-pipes/tika-pipes-pipe-iterators/tika-pipe-iterator-csv/pom.xml
index 056779c4b..ed76a8da0 100644
--- a/tika-pipes/tika-pipes-pipe-iterators/tika-pipe-iterator-csv/pom.xml
+++ b/tika-pipes/tika-pipes-pipe-iterators/tika-pipe-iterator-csv/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.tika</groupId>
<artifactId>tika-pipes-pipe-iterators</artifactId>
- <version>3.0.0-beta7</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>tika-pipe-iterator-csv</artifactId>
@@ -13,7 +13,7 @@
<description>A Tika Pipes Pipe Iterator that iterates through keys from a CSV on the file system.</description>
<properties>
<maven.compiler.release>17</maven.compiler.release>
- <commons-io.version>2.18.0</commons-io.version>
+ <commons-io.version>2.20.0</commons-io.version>
<commons-csv.version>1.13.0</commons-csv.version>
</properties>
<dependencies>
diff --git a/tika-pipes/tika-pipes-pipe-iterators/tika-pipe-iterator-csv/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-pipe-iterators/tika-pipe-iterator-csv/src/main/resources/plugin.properties
index 26c00c1c2..1e0e73675 100644
--- a/tika-pipes/tika-pipes-pipe-iterators/tika-pipe-iterator-csv/src/main/resources/plugin.properties
+++ b/tika-pipes/tika-pipes-pipe-iterators/tika-pipe-iterator-csv/src/main/resources/plugin.properties
@@ -16,6 +16,6 @@
plugin.id=csv-pipe-iterator
plugin.class=org.apache.tika.pipes.fetchers.filesystem.CsvPipeIteratorPlugin
-plugin.version=3.0.0-beta7
+plugin.version=4.0.0-SNAPSHOT
plugin.provider=CSV Pipe Iterator
plugin.description=Pipe iterator that reads CSV files and returns rows as batches.
diff --git a/tika-pipes/tika-pipes-pipe-iterators/tika-pipe-iterators-core/pom.xml b/tika-pipes/tika-pipes-pipe-iterators/tika-pipe-iterators-core/pom.xml
index 61e012d20..6eb94f8f0 100644
--- a/tika-pipes/tika-pipes-pipe-iterators/tika-pipe-iterators-core/pom.xml
+++ b/tika-pipes/tika-pipes-pipe-iterators/tika-pipe-iterators-core/pom.xml
@@ -13,7 +13,38 @@
<description>Tika Pipes Pipe Iterator library.</description>
<properties>
<maven.compiler.release>17</maven.compiler.release>
+ <protobuf.version>3.25.8</protobuf.version>
+ <j2objc-annotations.version>3.0.0</j2objc-annotations.version>
+ <grpc-google.version>2.51.0</grpc-google.version>
+ <failureaccess.version>1.0.2</failureaccess.version>
</properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <!-- Force consistent versions to resolve convergence issues -->
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java-util</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.j2objc</groupId>
+ <artifactId>j2objc-annotations</artifactId>
+ <version>${j2objc-annotations.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.api.grpc</groupId>
+ <artifactId>proto-google-common-protos</artifactId>
+ <version>${grpc-google.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>failureaccess</artifactId>
+ <version>${failureaccess.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<dependency>
<groupId>org.pf4j</groupId>
diff --git a/tika-pipes/tika-pipes-proto/pom.xml b/tika-pipes/tika-pipes-proto/pom.xml
index b08878754..a3d810603 100644
--- a/tika-pipes/tika-pipes-proto/pom.xml
+++ b/tika-pipes/tika-pipes-proto/pom.xml
@@ -12,37 +12,38 @@
<name>Tika Pipes GRPC Server Proto Stubs</name>
<properties>
<maven.compiler.release>17</maven.compiler.release>
- <grpc.version>1.70.0</grpc.version>
- <protobuf.version>3.24.0</protobuf.version>
- <protoc.version>3.24.0</protoc.version>
+ <grpc.version>1.72.0</grpc.version>
+ <protoc.version>3.25.8</protoc.version>
+ <!-- Use parent's protobuf version to resolve convergence issues -->
<asarkar-grpc-test.version>1.2.2</asarkar-grpc-test.version>
<awaitility.version>4.2.1</awaitility.version>
<j2objc-annotations.version>3.0.0</j2objc-annotations.version>
- <error_prone_annotations.version>2.28.0</error_prone_annotations.version>
- <jackson-module-jsonSchema.version>2.15.0</jackson-module-jsonSchema.version>
- <guava.version>33.4.0-jre</guava.version>
+ <error_prone_annotations.version>2.42.0</error_prone_annotations.version>
+ <jackson-module-jsonSchema.version>2.18.2</jackson-module-jsonSchema.version>
<spring-data-jpa.version>3.3.4</spring-data-jpa.version>
<ignite.version>2.16.0</ignite.version>
<ignite-spring-data-ext.version>3.0.0</ignite-spring-data-ext.version>
<jakarta.persistence-api.version>3.2.0</jakarta.persistence-api.version>
- <tika-version>3.0.0</tika-version>
+ <tika-version>4.0.0-SNAPSHOT</tika-version>
<grpc-google.version>2.51.0</grpc-google.version>
+ <checker-qual.version>3.47.0</checker-qual.version>
</properties>
+
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-parsers-standard-package</artifactId>
- <version>${tika-version}</version>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
- <version>${tika-version}</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-parser-microsoft-module</artifactId>
- <version>${tika-version}</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
@@ -52,7 +53,7 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>3.25.4</version>
+ <version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
@@ -63,6 +64,37 @@
<groupId>com.google.api.grpc</groupId>
<artifactId>grpc-google-cloud-translate-v3</artifactId>
<version>${grpc-google.version}</version>
+ <exclusions>
+ <!-- Exclude conflicting versions to use our managed versions -->
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf-lite</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.api.grpc</groupId>
+ <artifactId>proto-google-common-protos</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.j2objc</groupId>
+ <artifactId>j2objc-annotations</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.hsqldb</groupId>
@@ -131,6 +163,12 @@
<artifactId>jackson-module-jsonSchema</artifactId>
<version>${jackson-module-jsonSchema.version}</version>
</dependency>
+ <!-- Explicit dependency to force consistent version -->
+ <dependency>
+ <groupId>org.checkerframework</groupId>
+ <artifactId>checker-qual</artifactId>
+ <version>${checker-qual.version}</version>
+ </dependency>
</dependencies>
<build>
<extensions>
diff --git a/tika-server-old/tika-server-client/pom.xml b/tika-server-old/tika-server-client/pom.xml
deleted file mode 100644
index 1def8ca8c..000000000
--- a/tika-server-old/tika-server-client/pom.xml
+++ /dev/null
@@ -1,124 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-server</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-server-client</artifactId>
- <name>Apache Tika server client</name>
-
- <dependencies>
- <!-- TODO: Temporary comment out for tika-server migration to Spring Boot
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- -->
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-serialization</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-httpclient-commons</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>${httpcomponents.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.server.client</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>module-info.class</exclude>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.tika.server.client.TikaClientCLI</mainClass>
- </transformer>
-
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java b/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java
deleted file mode 100644
index 977b07477..000000000
--- a/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.client;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.tika.client.HttpClientFactory;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.serialization.JsonFetchEmitTuple;
-
-public class TikaClient {
-
- private final Random random = new Random();
- private final List<TikaPipesHttpClient> clients;
-
-
- private TikaClient(List<TikaPipesHttpClient> clients) {
- this.clients = clients;
- }
-
- public static TikaClient get(HttpClientFactory httpClientFactory, List<String> tikaServers) throws TikaConfigException {
- List clients = new ArrayList<>();
- for (String url : tikaServers) {
- //client factory is not thread safe, create a copy per client
- clients.add(new TikaPipesHttpClient(url, httpClientFactory.copy()));
- }
- return new TikaClient(clients);
- }
-
- /*public List<Metadata> parse(InputStream is, Metadata metadata)
- throws IOException, TikaException {
-
- }*/
-
- public TikaEmitterResult parse(FetchEmitTuple fetchEmit) throws IOException, TikaException {
- TikaPipesHttpClient client = getHttpClient();
- StringWriter writer = new StringWriter();
- JsonFetchEmitTuple.toJson(fetchEmit, writer);
- return client.postJson(writer.toString());
- }
-
- private TikaPipesHttpClient getHttpClient() {
- if (clients.size() == 1) {
- return clients.get(0);
- }
- int index = random.nextInt(clients.size());
- return clients.get(index);
- }
-}
diff --git a/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java b/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java
deleted file mode 100644
index f1cd718f1..000000000
--- a/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.client;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Collections;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.pipesiterator.CallablePipesIterator;
-import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
-
-public class TikaClientCLI {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TikaClientCLI.class);
- private static final int QUEUE_SIZE = 10000;
-
- public static void main(String[] args) throws Exception {
- Path tikaConfigPath = Paths.get(args[0]);
- TikaClientCLI cli = new TikaClientCLI();
- cli.execute(tikaConfigPath);
- }
-
- private void execute(Path tikaConfigPath) throws TikaException, IOException, SAXException {
- TikaServerClientConfig clientConfig = TikaServerClientConfig.build(tikaConfigPath);
-
- ExecutorService executorService = Executors.newFixedThreadPool(clientConfig.getNumThreads() + 1);
-
- ExecutorCompletionService<Long> completionService = new ExecutorCompletionService<>(executorService);
-
- final PipesIterator pipesIterator = PipesIterator.build(tikaConfigPath);
-
- final ArrayBlockingQueue<FetchEmitTuple> queue = new ArrayBlockingQueue<>(QUEUE_SIZE);
-
- completionService.submit(new CallablePipesIterator(pipesIterator, queue));
-
- if (clientConfig
- .getTikaEndpoints()
- .size() == clientConfig.getNumThreads()) {
- logDiffSizes(clientConfig
- .getTikaEndpoints()
- .size(), clientConfig.getNumThreads());
- for (int i = 0; i < clientConfig.getNumThreads(); i++) {
- TikaClient client = TikaClient.get(clientConfig.getHttpClientFactory(), Collections.singletonList(clientConfig
- .getTikaEndpoints()
- .get(i)));
- completionService.submit(new FetchWorker(queue, client, clientConfig.getMaxWaitMillis()));
- }
- } else {
- for (int i = 0; i < clientConfig.getNumThreads(); i++) {
- TikaClient client = TikaClient.get(clientConfig.getHttpClientFactory(), clientConfig.getTikaEndpoints());
- completionService.submit(new FetchWorker(queue, client, clientConfig.getMaxWaitMillis()));
- }
- }
-
- int finished = 0;
- while (finished < clientConfig.getNumThreads() + 1) {
- Future<Long> future = null;
- try {
- future = completionService.poll(30, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- //stop the world
- LOGGER.error("", e);
- throw new RuntimeException(e);
- }
- if (future != null) {
- finished++;
- try {
- future.get();
- } catch (InterruptedException | ExecutionException e) {
- //stop the world
- LOGGER.error("critical main loop failure", e);
- throw new RuntimeException(e);
- }
- }
- }
- }
-
- private void logDiffSizes(int servers, int numThreads) {
- LOGGER.info("tika server count ({}) != numThreads ({}). " + "Each client will randomly select a server from this list", servers, numThreads);
- }
-
- private static class FetchWorker implements Callable<Long> {
- private final ArrayBlockingQueue<FetchEmitTuple> queue;
- private final TikaClient client;
-
- private final long maxWaitMs;
-
- public FetchWorker(ArrayBlockingQueue<FetchEmitTuple> queue, TikaClient client, long maxWaitMs) {
- this.queue = queue;
- this.client = client;
- this.maxWaitMs = maxWaitMs;
- }
-
- @Override
- public Long call() throws Exception {
- while (true) {
-
- FetchEmitTuple t = queue.poll(maxWaitMs, TimeUnit.MILLISECONDS);
- if (t == null) {
- throw new TimeoutException("exceeded maxWaitMs");
- }
- if (t == PipesIterator.COMPLETED_SEMAPHORE) {
- //potentially blocks forever
- queue.put(PipesIterator.COMPLETED_SEMAPHORE);
- return 1l;
- }
- try {
- LOGGER.debug("about to parse: {}", t.getFetchKey());
- client.parse(t);
- } catch (IOException | TikaException e) {
- LOGGER.warn(t
- .getFetchKey()
- .toString(), e);
- }
- }
- }
- }
-}
diff --git a/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientConfigException.java b/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientConfigException.java
deleted file mode 100644
index 97776b8bc..000000000
--- a/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientConfigException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.client;
-
-import org.apache.tika.exception.TikaException;
-
-public class TikaClientConfigException extends TikaException {
-
- public TikaClientConfigException(String msg) {
- super(msg);
- }
-
- public TikaClientConfigException(String msg, Throwable cause) {
- super(msg, cause);
- }
-}
diff --git a/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java b/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java
deleted file mode 100644
index 63b15fd1b..000000000
--- a/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaEmitterResult.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.client;
-
-public class TikaEmitterResult {
-
-
- private final STATUS status;
- private final String msg;//used for exceptions. will be null for status ok
- private final long timeElapsed;
-
- public TikaEmitterResult(STATUS status, long timeElapsed, String msg) {
- this.status = status;
- this.timeElapsed = timeElapsed;
- this.msg = msg;
- }
-
- @Override
- public String toString() {
- return "TikaEmitterResult{" + "status=" + status + ", msg='" + msg + '\'' + ", timeElapsed=" + timeElapsed + '}';
- }
-
- public STATUS getStatus() {
- return status;
- }
-
- public String getMsg() {
- return msg;
- }
-
- public long getTimeElapsed() {
- return timeElapsed;
- }
-
- enum STATUS {
- OK, NOT_OK, EXCEEDED_MAX_RETRIES, TIMED_OUT_WAITING_FOR_TIKA
- }
-
-
-}
diff --git a/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaPipesHttpClient.java b/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaPipesHttpClient.java
deleted file mode 100644
index 83edf2d8c..000000000
--- a/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaPipesHttpClient.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.client;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.client.HttpClientFactory;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-
-/**
- * Low-level class to handle the http layer.
- */
-class TikaPipesHttpClient {
-
- private static final String TIKA_ENDPOINT = "tika";
- private static final Logger LOGGER = LoggerFactory.getLogger(TikaPipesHttpClient.class);
- private final String endPoint = "pipes";
-
- private final HttpClientFactory httpClientFactory;
- private final String endPointUrl;
- private final String tikaUrl;
- private final int maxRetries = 3;
- //if can't make contact with Tika server, max wait time in ms
- private final long maxWaitForTikaMs = 120000;
- //how often to ping /tika (in ms) to see if the server is up and running
- private final long pulseWaitForTikaMs = 1000;
- private HttpClient httpClient;
-
- /**
- * @param baseUrl url to base endpoint
- */
- TikaPipesHttpClient(String baseUrl, HttpClientFactory httpClientFactory) throws TikaConfigException {
- if (!baseUrl.endsWith("/")) {
- baseUrl += "/";
- }
- this.endPointUrl = baseUrl + getEndpoint();
- this.tikaUrl = baseUrl + TIKA_ENDPOINT;
- this.httpClientFactory = httpClientFactory;
- this.httpClient = getNewClient(baseUrl);
- }
-
- String getEndpoint() {
- return endPoint;
- }
-
- private HttpClient getNewClient(String baseUrl) throws TikaConfigException {
- if (httpClient instanceof CloseableHttpClient) {
- try {
- ((CloseableHttpClient) httpClient).close();
- } catch (IOException e) {
- LOGGER.warn("exception closing client", e);
- }
- }
- return httpClientFactory.build();
- }
-
-
- public TikaEmitterResult postJson(String jsonRequest) {
- return postJson(endPointUrl, jsonRequest);
- }
-
- private TikaEmitterResult postJson(String endPoint, String jsonRequest) {
- HttpPost post = new HttpPost(endPoint);
- post.setEntity(new StringEntity(jsonRequest, StandardCharsets.UTF_8));
- post.setHeader("Content-Type", "application/json");
-
- int tries = 0;
- long start = System.currentTimeMillis();
- try {
- while (tries++ < maxRetries) {
- HttpResponse response = null;
- try {
- response = httpClient.execute(post);
- } catch (IOException e) {
- LOGGER.warn("Exception trying to parse", e);
- waitForServer();
- continue;
- }
- String msg = "";
- try {
- msg = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
- } catch (IOException e) {
- //swallow
- }
- long elapsed = System.currentTimeMillis() - start;
- TikaEmitterResult.STATUS status = TikaEmitterResult.STATUS.OK;
- if (response
- .getStatusLine()
- .getStatusCode() != 200) {
- status = TikaEmitterResult.STATUS.NOT_OK;
- } else {
- //pull out stacktrace from parse exception?
- }
- return new TikaEmitterResult(status, elapsed, msg);
- }
- } catch (TimeoutWaitingForTikaException e) {
- long elapsed = System.currentTimeMillis() - start;
- return new TikaEmitterResult(TikaEmitterResult.STATUS.TIMED_OUT_WAITING_FOR_TIKA, elapsed, "");
- }
- long elapsed = System.currentTimeMillis() - start;
- return new TikaEmitterResult(TikaEmitterResult.STATUS.EXCEEDED_MAX_RETRIES, elapsed, "");
- }
-
-
- private void waitForServer() throws TimeoutWaitingForTikaException {
- long start = System.currentTimeMillis();
- long elapsed = System.currentTimeMillis() - start;
- LOGGER.info("server unreachable; waiting for it to restart");
- while (elapsed < maxWaitForTikaMs) {
- try {
- Thread.sleep(pulseWaitForTikaMs);
- } catch (InterruptedException e) {
- //swallow
- }
-
- HttpGet get = new HttpGet(tikaUrl);
- try {
- HttpResponse response = httpClient.execute(get);
- if (response
- .getStatusLine()
- .getStatusCode() == 200) {
- LOGGER.debug("server back up");
- return;
- }
- } catch (IOException e) {
- elapsed = System.currentTimeMillis() - start;
- LOGGER.debug("waiting for server; failed to reach it: {} ms", elapsed);
- }
-
- elapsed = System.currentTimeMillis() - start;
- }
-
- LOGGER.warn("Timeout waiting for tika server {} in {} ms", tikaUrl, elapsed);
- throw new TimeoutWaitingForTikaException("");
- }
-
- static class TimeoutWaitingForTikaException extends TikaException {
- public TimeoutWaitingForTikaException(String msg) {
- super(msg);
- }
- }
-}
diff --git a/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaServerClientConfig.java b/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaServerClientConfig.java
deleted file mode 100644
index 01a6d16e1..000000000
--- a/tika-server-old/tika-server-client/src/main/java/org/apache/tika/server/client/TikaServerClientConfig.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.client;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.tika.client.HttpClientFactory;
-import org.apache.tika.config.ConfigBase;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-
-public class TikaServerClientConfig extends ConfigBase implements Initializable {
-
-
- private HttpClientFactory httpClientFactory;
- private int numThreads = 1;
- private MODE mode = MODE.PIPES;
- private List<String> tikaEndpoints = new ArrayList<>();
- private long maxWaitMillis = 60000;
-
- public static TikaServerClientConfig build(Path configFile) throws IOException, TikaConfigException {
- try (InputStream is = Files.newInputStream(configFile)) {
- return buildSingle("serverClientConfig", TikaServerClientConfig.class, is);
- }
- }
-
- public long getMaxWaitMillis() {
- return maxWaitMillis;
- }
-
- /**
- * maximum time in milliseconds to wait for a new fetchemittuple to be
- * available from the queue. The client will end if no tuple is available
- * within this amount of time.
- *
- * @param maxWaitMs
- */
- public void setMaxWaitMillis(long maxWaitMs) {
- this.maxWaitMillis = maxWaitMs;
- }
-
- public HttpClientFactory getHttpClientFactory() {
- return httpClientFactory;
- }
-
- public void setHttpClientFactory(HttpClientFactory httpClientFactory) {
- this.httpClientFactory = httpClientFactory;
- }
-
- public int getNumThreads() {
- return numThreads;
- }
-
- public void setNumThreads(int numThreads) {
- this.numThreads = numThreads;
- }
-
- public MODE getMode() {
- return mode;
- }
-
- public void setMode(String mode) {
- if ("pipes".equals(mode)) {
- this.mode = MODE.PIPES;
- return;
- }
- throw new IllegalArgumentException("I regret that we have not yet implemented: '" + mode + "'");
- }
-
- public List<String> getTikaEndpoints() {
- return tikaEndpoints;
- }
-
- public void setTikaEndpoints(List<String> tikaEndpoints) {
- this.tikaEndpoints = tikaEndpoints;
- }
-
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
-
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
- if (tikaEndpoints.size() == 0) {
- throw new TikaConfigException("tikaEndpoints must not be empty");
- }
- }
-
- enum MODE {
- PIPES, ASYNC
- }
-}
diff --git a/tika-server-old/tika-server-client/src/test/java/org/apache/tika/server/client/TestBasic.java b/tika-server-old/tika-server-client/src/test/java/org/apache/tika/server/client/TestBasic.java
deleted file mode 100644
index 78d76c7cc..000000000
--- a/tika-server-old/tika-server-client/src/test/java/org/apache/tika/server/client/TestBasic.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.client;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-public class TestBasic {
-
- @Test
- public void testConfig() throws Exception {
- Path p = Paths.get(TestBasic.class
- .getResource("/tika-config-simple-fs-emitter.xml")
- .toURI());
- assertTrue(Files.isRegularFile(p));
-
- TikaServerClientConfig clientConfig = TikaServerClientConfig.build(p);
- assertEquals(6, clientConfig.getNumThreads());
- assertEquals(5, clientConfig
- .getHttpClientFactory()
- .getMaxConnections());
- }
-
- @Test
- @Disabled("turn this into an actual test in tika-integration-tests?")
- public void testBasic() throws Exception {
- Path p = Paths.get(TestBasic.class
- .getResource("/tika-config-simple-fs-emitter.xml")
- .toURI());
- assertTrue(Files.isRegularFile(p));
- String[] args = new String[]{p.toAbsolutePath().toString()};
- long start = System.currentTimeMillis();
- TikaClientCLI.main(args);
- long elapsed = System.currentTimeMillis() - start;
- System.out.println("elapsed " + elapsed);
- }
-}
diff --git a/tika-server-old/tika-server-client/src/test/resources/tika-config-simple-fs-emitter.xml b/tika-server-old/tika-server-client/src/test/resources/tika-config-simple-fs-emitter.xml
deleted file mode 100644
index b2e516362..000000000
--- a/tika-server-old/tika-server-client/src/test/resources/tika-config-simple-fs-emitter.xml
+++ /dev/null
@@ -1,106 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <service-loader initializableProblemHandler="throw"/>
- <parsers>
- <parser class="org.apache.tika.parser.DefaultParser">
- <parser-exclude class="org.apache.tika.parser.ocr.TesseractOCRParser"/>
- </parser>
- </parsers>
- <pipesIterator class="org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator">
- <params>
- <fetcherName>fs</fetcherName>
- <emitterName>fs</emitterName>
- <basePath>UPDATE</basePath>
- </params>
- </pipesIterator>
- <fetchers>
- <fetcher class="org.apache.tika.pipes.fetcher.fs.FileSystemFetcher">
- <params>
- <name>fs</name>
- <basePath>UPDATE</basePath>
- </params>
- </fetcher>
- </fetchers>
- <metadataFilters>
- <metadataFilter class="org.apache.tika.metadata.filter.FieldNameMappingFilter">
- <params>
- <excludeUnmapped>true</excludeUnmapped>
- <mappings>
- <mapping from="X-TIKA:content" to="content"/>
- <mapping from="X-TIKA:embedded_resource_path" to="embedded_path"/>
- <mapping from="Content-Length" to="length"/>
- <mapping from="dc:creator" to="creators"/>
- <mapping from="dc:title" to="title"/>
- </mappings>
- </params>
- </metadataFilter>
- </metadataFilters>
- <emitters>
- <emitter class="org.apache.tika.pipes.core.emitter.fs.FileSystemEmitter">
- <params>
- <name>fs</name>
- <basePath>UPDATE</basePath>
- </params>
- </emitter>
- </emitters>
- <pipes>
- <params>
- <numClients>6</numClients>
- <forkedJvmArgs>
- <arg>-Xmx1g</arg>
- <arg>-XX:ParallelGCThreads=2</arg>
- </forkedJvmArgs>
- <timeoutMillis>60000</timeoutMillis>
- </params>
- </pipes>
- <server>
- <params>
- <port>9998</port>
- <host>localhost</host>
- <digest>sha256</digest>
- <digestMarkLimit>1000000</digestMarkLimit>
- <logLevel>debug</logLevel>
- <returnStackTrace>true</returnStackTrace>
- <noFork>true</noFork>
- <endpoints>
- <endpoint>tika</endpoint>
- <endpoint>pipes</endpoint>
- </endpoints>
- </params>
- </server>
- <serverClientConfig>
- <params>
- <!-- this is the only option so far -->
- <mode>pipes</mode>
- <!-- this should be the same number as that in the pipes' numClients -->
- <numThreads>6</numThreads>
- <maxWaitMillis>600000</maxWaitMillis>
- <tikaEndpoints>
- <tikaEndpoint>http://localhost:9998</tikaEndpoint>
- </tikaEndpoints>
- </params>
- <httpClientFactory class="org.apache.tika.client.HttpClientFactory">
- <params>
- <maxConnections>5</maxConnections>
- </params>
- </httpClientFactory>
- </serverClientConfig>
-</properties>
\ No newline at end of file
diff --git a/tika-server-old/tika-server-core/pom.xml b/tika-server-old/tika-server-core/pom.xml
deleted file mode 100644
index f22998756..000000000
--- a/tika-server-old/tika-server-core/pom.xml
+++ /dev/null
@@ -1,334 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-server</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
-
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-server-core</artifactId>
- <name>Apache Tika server core</name>
-
- <pluginRepositories>
- <pluginRepository>
- <id>miredot</id>
- <name>MireDot Releases</name>
- <url>http://nexus.qmino.com/content/repositories/miredot</url>
- </pluginRepository>
- </pluginRepositories>
-
- <dependencies>
- <!-- TODO: Temporary comment out for tika-server migration to Spring Boot
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- -->
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-translate</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-langdetect-optimaize</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-serialization</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-handler-boilerpipe</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-parser-digest-commons</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-parser-zip-commons</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-csv</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.cxf</groupId>
- <artifactId>cxf-rt-frontend-jaxrs</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.cxf</groupId>
- <artifactId>cxf-rt-rs-service-description</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.cxf</groupId>
- <artifactId>cxf-rt-transports-http-jetty</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.cxf</groupId>
- <artifactId>cxf-rt-rs-security-cors</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.cxf</groupId>
- <artifactId>cxf-rt-rs-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <!-- logging -->
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <redirectTestOutputToFile>true</redirectTestOutputToFile>
- <argLine>-da -XX:+HeapDumpOnOutOfMemoryError -Xmx512m</argLine>
- <systemPropertyVariables>
- <java.util.logging.config.file>
- ${basedir}/src/main/resources/commons-logging.properties
- </java.util.logging.config.file>
- </systemPropertyVariables>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>module-info.class</exclude>
- <exclude>META-INF/maven/plugin.xml</exclude>
- <exclude>META-INF/versions/9/module-info.class</exclude>
- <exclude>META-INF/versions/11/module-info.class</exclude>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- <exclude>META-INF/*.txt</exclude>
- <exclude>META-INF/ASL2.0</exclude>
- <exclude>META-INF/DEPENDENCIES</exclude>
- <exclude>META-INF/LICENSE</exclude>
- <exclude>META-INF/NOTICE</exclude>
- <exclude>META-INF/LICENSE.md</exclude>
- <exclude>META-INF/NOTICE.md</exclude>
- <exclude>META-INF/README</exclude>
- <exclude>META-INF/MANIFEST.MF</exclude>
- <exclude>about.html</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- <exclude>CHANGES</exclude>
- <exclude>README</exclude>
- <exclude>builddef.lst</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.tika.server.core.TikaServerCli</mainClass>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>META-INF/blueprint.handlers</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>META-INF/spring.handlers</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>META-INF/spring.schemas</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>META-INF/cxf/cxf.extension</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
- <resource>META-INF/extensions.xml</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
- <resource>META-INF/cxf/extensions.xml</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>META-INF/cxf/bus-extensions.txt</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
- <resource>META-INF/cxf/bus-extensions.xml</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
- <resource>META-INF/wsdl.plugin.xml</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
- <resource>META-INF/tools.service.validator.xml</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
- <resource>META-INF/cxf/java2wsbeans.xml</resource>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.server.core</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${rat.version}</version>
- <configuration>
- <excludes>
- <exclude>src/main/resources/tikaserver-version.properties</exclude>
- <exclude>src/test/resources/*</exclude>
- <exclude>src/test/resources/test-documents/**</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </build>
- <profiles>
- <profile>
- <id>server</id>
- <build>
- <defaultGoal>test</defaultGoal>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>3.5.1</version>
- <executions>
- <execution>
- <phase>test</phase>
- <goals>
- <goal>java</goal>
- </goals>
- <configuration>
- <mainClass>org.apache.tika.server.core.TikaServerCli</mainClass>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>com.qmino</groupId>
- <artifactId>miredot-plugin</artifactId>
- <version>2.4.5-Java11</version>
- <executions>
- <execution>
- <goals>
- <goal>restdoc</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <organizationId>c579ca2e-c194-4a14-a7f0-f39ef4ec3c2d</organizationId>
- <restModel>
- <restFramework>
- <name>jax-rs</name>
- </restFramework>
- </restModel>
- <output>
- <html />
- </output>
- <!-- Free Miredot license key, valid until Jan 31st(?), 2025
- https://issues.apache.org/jira/browse/TIKA-2253
- https://issues.apache.org/jira/browse/TIKA-3038-->
- <licence>
- cHJvamVjdHxvcmcuYXBhY2hlLnRpa2EudGlrYS1zZXJ2ZXJ8MjAyNS0wMy0wMXxmYWxzZXwtMSNNQ3dDRkdtemsraHJDdjNodXlhN0d1KzlQM1Y0bUNxMEFoUnRHc25DQ1VyMjhLQ2swU0R5RGp1WkVpcXVlZz09
- </licence>
- <!-- insert other configuration here (optional) -->
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/CompositeParseContextConfig.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/CompositeParseContextConfig.java
deleted file mode 100644
index 29ab876b1..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/CompositeParseContextConfig.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core;
-
-import java.util.List;
-
-import jakarta.ws.rs.core.MultivaluedMap;
-
-import org.apache.tika.config.ServiceLoader;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-
-public class CompositeParseContextConfig implements ParseContextConfig {
-
-
- final List<ParseContextConfig> configs;
-
- public CompositeParseContextConfig() {
- configs = new ServiceLoader(CompositeParseContextConfig.class.getClassLoader()).loadServiceProviders(ParseContextConfig.class);
- }
-
- @Override
- public void configure(MultivaluedMap<String, String> httpHeaders, Metadata metadata, ParseContext context) {
- for (ParseContextConfig config : configs) {
- config.configure(httpHeaders, metadata, context);
- }
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/FetcherStreamFactory.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/FetcherStreamFactory.java
deleted file mode 100644
index 66e4b69fc..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/FetcherStreamFactory.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
-import java.nio.charset.StandardCharsets;
-
-import jakarta.ws.rs.core.HttpHeaders;
-import jakarta.ws.rs.core.MultivaluedMap;
-import jakarta.ws.rs.core.UriInfo;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.Fetcher;
-import org.apache.tika.pipes.core.fetcher.FetcherManager;
-import org.apache.tika.pipes.core.fetcher.RangeFetcher;
-import org.apache.tika.server.core.resource.TikaResource;
-
-/**
- * This class looks for "fetcherName" in the http header. If it is not null
- * and not empty, this will return a new TikaInputStream from the fetch key
- * and the base path as set in the definition of the named fetcher.
- * As of Tika > 2.5.0, the "fetchKey" is URL decoded.
- * <p>
- * Users may also specify the "fetcherName"e; and "fetchKey" in
- * query parameters with in the request.
- * <p>
- * <em>WARNING:</em> Unless you carefully lock down access to the server,
- * whoever has access to this service will have the read access of the server.
- * In short, anyone with access to this service could request and get
- * "file:///etc/supersensitive_file_dont_read.txt". Or, if your server has access
- * to your intranet, and you let the public hit this service, they will now
- * have access to your intranet.
- * See <a href="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-3271">CVE-2015-3271</a>
- */
-public class FetcherStreamFactory implements InputStreamFactory {
-
- private static final Logger LOG = LoggerFactory.getLogger(FetcherStreamFactory.class);
-
- private final FetcherManager fetcherManager;
-
- public FetcherStreamFactory(FetcherManager fetcherManager) {
- this.fetcherManager = fetcherManager;
- }
-
- /**
- * Tries to parse a long out of the value. If the val is blank, it returns -1.
- * Throws {@link NumberFormatException}
- *
- * @param val
- * @return
- */
- private static long getLong(String val) {
- if (StringUtils.isBlank(val)) {
- return -1;
- }
- return Long.parseLong(val);
- }
-
- @Override
- public InputStream getInputStream(InputStream is, Metadata metadata, HttpHeaders httpHeaders, UriInfo uriInfo) throws IOException {
- MultivaluedMap params = (uriInfo == null) ? null : uriInfo.getQueryParameters();
- String fetcherName = getParam("fetcherName", httpHeaders, params);
- String fetchKey = getParam("fetchKey", httpHeaders, params);
- fetchKey = urlDecode(fetchKey);
- if (StringUtils.isBlank(fetchKey)) {
- fetchKey = getParam("fetchKeyLiteral", httpHeaders, params);
- }
- ParseContext parseContext = new ParseContext();
- TikaResource.fillParseContext(httpHeaders.getRequestHeaders(), metadata, parseContext);
- long fetchRangeStart = getLong(getParam("fetchRangeStart", httpHeaders, params));
- long fetchRangeEnd = getLong(getParam("fetchRangeEnd", httpHeaders, params));
- if (StringUtils.isBlank(fetcherName) != StringUtils.isBlank(fetchKey)) {
- throw new IOException("Must specify both a 'fetcherName' and a 'fetchKey'. I see: " + " fetcherName:" + fetcherName + " and fetchKey:" + fetchKey);
- }
- if (fetchRangeStart < 0 && fetchRangeEnd > -1) {
- throw new IllegalArgumentException("fetchRangeStart must be > -1 if a fetchRangeEnd " + "is specified");
- }
-
- if (fetchRangeStart > -1 && fetchRangeEnd < 0) {
- throw new IllegalArgumentException("fetchRangeEnd must be > -1 if a fetchRangeStart " + "is specified");
- }
-
- if (!StringUtils.isBlank(fetcherName)) {
- try {
- LOG.debug("going to fetch '{}' from fetcher: {}", fetchKey, fetcherName);
- Fetcher fetcher = fetcherManager.getFetcher(fetcherName);
- if (fetchRangeStart > -1 && fetchRangeEnd > -1 && !(fetcher instanceof RangeFetcher)) {
- throw new IllegalArgumentException(
- "Can't call a fetch with a range on a fetcher that" + " is not a RangeFetcher: name=" + fetcher.getName() + " class=" + fetcher.getClass());
- }
- return fetcher.fetch(fetchKey, metadata, parseContext);
- } catch (TikaException e) {
- throw new IOException(e);
- }
- }
- return is;
- }
-
- private String urlDecode(String fetchKey) {
- if (fetchKey == null) {
- return fetchKey;
- }
- try {
- return URLDecoder.decode(fetchKey, StandardCharsets.UTF_8.toString());
- } catch (UnsupportedEncodingException | IllegalArgumentException e) {
- LOG.warn("couldn't decode fetch key {}", fetchKey);
- }
- return fetchKey;
- }
-
- private String getParam(String paramName, HttpHeaders httpHeaders, MultivaluedMap uriParams) {
- if (uriParams == null || !uriParams.containsKey(paramName)) {
- return httpHeaders.getHeaderString(paramName);
- }
-
- return (String) uriParams.getFirst(paramName);
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/HTMLHelper.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/HTMLHelper.java
deleted file mode 100644
index 631545a1a..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/HTMLHelper.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.commons.io.IOUtils;
-
-/**
- * Helps produce user facing HTML output.
- * <p/>
- * TODO Decide if this would be better done as a MessageBodyWriter
- */
-public class HTMLHelper {
- private static final String PATH = "/tikaserver-template.html";
- private static final String TITLE_VAR = "[[TITLE]]";
- private static final String BODY_VAR = "[[BODY]]";
- private String PRE_BODY;
- private String POST_BODY;
-
- public HTMLHelper() {
- try (InputStream htmlStr = getClass().getResourceAsStream(PATH)) {
- if (htmlStr == null) {
- throw new IllegalArgumentException("Template Not Found - " + PATH);
- }
-
- String html = IOUtils.toString(htmlStr, UTF_8);
- int bodyAt = html.indexOf(BODY_VAR);
- PRE_BODY = html.substring(0, bodyAt);
- POST_BODY = html.substring(bodyAt + BODY_VAR.length());
-
- } catch (IOException e) {
- throw new IllegalStateException("Unable to read template");
- }
- }
-
- /**
- * Generates the HTML Header for the user facing page, adding
- * in the given title as required
- */
- public void generateHeader(StringBuffer html, String title) {
- html.append(PRE_BODY.replace(TITLE_VAR, title));
- }
-
- public void generateFooter(StringBuffer html) {
- html.append(POST_BODY);
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/InputStreamFactory.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/InputStreamFactory.java
deleted file mode 100644
index 07d8dbaaa..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/InputStreamFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import jakarta.ws.rs.core.HttpHeaders;
-import jakarta.ws.rs.core.UriInfo;
-
-import org.apache.tika.metadata.Metadata;
-
-/**
- * Interface to allow for custom/consistent creation of InputStream
- * <p>
- * This factory is used statically in TikaResource. Make sure not
- * to hold instance state in implementations.
- */
-public interface InputStreamFactory {
-
- InputStream getInputStream(InputStream is, Metadata metadata, HttpHeaders httpHeaders, UriInfo uriInfo) throws IOException;
-
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ParseContextConfig.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ParseContextConfig.java
deleted file mode 100644
index 1ed656c39..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ParseContextConfig.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core;
-
-import jakarta.ws.rs.core.MultivaluedMap;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-
-/**
- * Implementations must be thread-safe!
- * <p>
- * This class translates http headers into objects/configurations set
- * via the ParseContext
- */
-public interface ParseContextConfig {
-
- /**
- * Configures the parseContext with present headers.
- *
- * @param headers the headers.
- * @param metadata the metadata.
- * @param context the parse context to configure.
- */
- void configure(MultivaluedMap<String, String> headers, Metadata metadata, ParseContext context);
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ProduceTypeResourceComparator.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ProduceTypeResourceComparator.java
deleted file mode 100644
index 53b120059..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ProduceTypeResourceComparator.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core;
-
-import java.util.Arrays;
-import java.util.List;
-
-import jakarta.ws.rs.core.MediaType;
-import org.apache.cxf.jaxrs.ext.ResourceComparator;
-import org.apache.cxf.jaxrs.model.ClassResourceInfo;
-import org.apache.cxf.jaxrs.model.OperationResourceInfo;
-import org.apache.cxf.jaxrs.utils.JAXRSUtils;
-import org.apache.cxf.message.Message;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Resource comparator based to produce type.
- * In an ambiguous call, request handler will be chosen based on the type of data it returns.
- */
-public class ProduceTypeResourceComparator implements ResourceComparator {
-
- /**
- * The prioritized MediaType list.
- * The lower in list (higher index value), the higher priority it has.
- * In case of no matching in this list, it will be treated as media type all.
- */
- public static final List<MediaType> PRIORITIZED_MEDIA_LIST =
- Arrays.asList(MediaType.TEXT_PLAIN_TYPE, MediaType.APPLICATION_JSON_TYPE, MediaType.TEXT_HTML_TYPE, MediaType.TEXT_XML_TYPE);
- /**
- * The logger
- */
- private static final Logger LOG = LoggerFactory.getLogger(ProduceTypeResourceComparator.class);
-
- /**
- * Initiates the comparator.
- * Currently, no op.
- */
- public ProduceTypeResourceComparator() {
- }
-
- /**
- * Compares the class to handle.
- * Nothing is handled here, and this is handed over to CXF default logic.
- *
- * @param cri1 the first class resource info.
- * @param cri2 the second class resource info.
- * @param message the message, for comparison context.
- * @return value based on chosen class. (Always 0 is returned)
- */
- @Override
- public int compare(ClassResourceInfo cri1, ClassResourceInfo cri2, Message message) {
- return 0;
- }
-
- /**
- * Compares the method to handle.
- * Gets the maximum priority match for both handlers,
- * and chooses the handler that has the maximum priority match.
- *
- * @param oper1 the first resource handler info.
- * @param oper2 the second resource handler info.
- * @param message the message, for comparison context.
- * @return value based on chosen handler. Returns -1 if first, 1 if second and 0 if no decision.
- */
- @Override
- public int compare(OperationResourceInfo oper1, OperationResourceInfo oper2, Message message) {
- // getting all message context data
- final String httpMethod = (String) message.get(Message.HTTP_REQUEST_METHOD);
- final MediaType contentType = JAXRSUtils.toMediaType((String) message.get(Message.CONTENT_TYPE));
- final List<MediaType> acceptTypes = JAXRSUtils.parseMediaTypes((String) message.get(Message.ACCEPT_CONTENT_TYPE));
-
- LOG.debug("Message Method : " + httpMethod + ", ContentType : " + contentType + ", " + "Accept Types : " + acceptTypes);
-
- int result = compareProduceTypes(oper1, oper2, acceptTypes);
-
- String m1Name = oper1
- .getClassResourceInfo()
- .getServiceClass()
- .getName() + "#" + oper1
- .getMethodToInvoke()
- .getName();
- String m2Name = oper2
- .getClassResourceInfo()
- .getServiceClass()
- .getName() + "#" + oper2
- .getMethodToInvoke()
- .getName();
-
- if (result != 0) {
- String chosen = result == -1 ? m1Name : m2Name;
- LOG.debug("Between " + m1Name + " and " + m2Name + ", " + chosen + " is chosen for handling the current request");
- }
-
- return result;
- }
-
- /**
- * Compares the method to handle.
- * Gets the maximum priority match for both handlers,
- * and chooses the handler that has the maximum priority match.
- *
- * @param oper1 the first resource handler info.
- * @param oper2 the second resource handler info.
- * @param acceptTypes the list acceptable response mime type, for the message.
- * @return value based on chosen handler. Returns -1 if first, 1 if second and 0 if no decision.
- */
- private int compareProduceTypes(OperationResourceInfo oper1, OperationResourceInfo oper2, final List<MediaType> acceptTypes) {
- // getting matched produce type for both handlers.
- // this is required if a method can produce multiple types.
- List<MediaType> op1Matched = JAXRSUtils.intersectMimeTypes(acceptTypes, oper1.getProduceTypes(), true);
- List<MediaType> op2Matched = JAXRSUtils.intersectMimeTypes(acceptTypes, oper2.getProduceTypes(), true);
-
- // calculate the max priority for both handlers
- int oper1Priority = op1Matched
- .stream()
- .mapToInt(PRIORITIZED_MEDIA_LIST::indexOf)
- .max()
- .getAsInt();
- int oper2Priority = op2Matched
- .stream()
- .mapToInt(PRIORITIZED_MEDIA_LIST::indexOf)
- .max()
- .getAsInt();
-
- // final calculation
- return oper1Priority == oper2Priority ? 0 : (oper1Priority > oper2Priority ? -1 : 1);
- }
-
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatus.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatus.java
deleted file mode 100644
index b53683de5..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatus.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core;
-
-import java.time.Instant;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ServerStatus {
-
- private static final Logger LOG = LoggerFactory.getLogger(ServerStatus.class);
- private final String serverId;
- private final int numRestarts;
-
- private final boolean isLegacy;
- private AtomicLong counter = new AtomicLong(0);
- private Map<Long, TaskStatus> tasks = new HashMap<>();
- private STATUS status = STATUS.OPERATING;
- private volatile long lastStarted = Instant
- .now()
- .toEpochMilli();
-
- public ServerStatus(String serverId, int numRestarts) {
- this(serverId, numRestarts, false);
- }
-
- public ServerStatus(String serverId, int numRestarts, boolean isLegacy) {
- this.serverId = serverId;
- this.numRestarts = numRestarts;
- this.isLegacy = isLegacy;
- }
-
- public synchronized long start(TASK task, String fileName, long timeoutMillis) {
- long taskId = counter.incrementAndGet();
- Instant now = Instant.now();
- lastStarted = now.toEpochMilli();
- tasks.put(taskId, new TaskStatus(task, now, fileName, timeoutMillis));
- return taskId;
- }
-
- /**
- * Removes the task from the collection of currently running tasks.
- *
- * @param taskId
- * @throws IllegalArgumentException if there is no task by that taskId in the collection
- */
- public synchronized void complete(long taskId) throws IllegalArgumentException {
- TaskStatus status = tasks.remove(taskId);
- if (status == null) {
- throw new IllegalArgumentException("TaskId is not in map:" + taskId);
- }
- }
-
- public synchronized STATUS getStatus() {
- return status;
- }
-
- public synchronized void setStatus(STATUS status) {
- this.status = status;
- }
-
- public synchronized Map<Long, TaskStatus> getTasks() {
- return new HashMap<>(tasks);
- }
-
- public synchronized long getFilesProcessed() {
- return counter.get();
- }
-
- public long getMillisSinceLastParseStarted() {
- return Instant
- .now()
- .toEpochMilli() - lastStarted;
- }
-
- /**
- * @return true if this is legacy, otherwise whether or not status == OPERATING.
- */
- public synchronized boolean isOperating() {
- if (isLegacy) {
- return true;
- }
- return status == STATUS.OPERATING;
- }
-
- public String getServerId() {
- return serverId;
- }
-
- public int getNumRestarts() {
- return numRestarts;
- }
-
- enum DIRECTIVES {
- PING((byte) 0), PING_ACTIVE_SERVER_TASKS((byte) 1), SHUTDOWN((byte) 2);
-
- private final byte b;
-
- DIRECTIVES(byte b) {
- this.b = b;
- }
-
- byte getByte() {
- return b;
- }
- }
-
- public enum STATUS {
- INITIALIZING(0), OPERATING(1), HIT_MAX_FILES(2), TIMEOUT(3), ERROR(4), PARENT_REQUESTED_SHUTDOWN(5), PARENT_EXCEPTION(6), OFF(7);
-
- private final int shutdownCode;
-
- STATUS(int shutdownCode) {
- this.shutdownCode = shutdownCode;
- }
-
- static STATUS lookup(int i) {
- STATUS[] values = STATUS.values();
- if (i < 0 || i >= values.length) {
- throw new ArrayIndexOutOfBoundsException(i + " is not acceptable for an array of length " + values.length);
- }
- return STATUS.values()[i];
- }
-
- int getShutdownCode() {
- return shutdownCode;
- }
-
- int getInt() {
- return shutdownCode;
- }
-
- }
-
- public enum TASK {
- PARSE, DETECT, TRANSLATE
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusResource.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusResource.java
deleted file mode 100644
index f5f8c8a78..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusResource.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core;
-
-import org.apache.tika.server.core.resource.TikaServerResource;
-
-public interface ServerStatusResource extends TikaServerResource {
- void setServerStatus(ServerStatus serverStatus);
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
deleted file mode 100644
index 55ce47a9b..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.Buffer;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileLock;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-import java.time.Duration;
-import java.time.Instant;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.server.core.config.TimeoutConfig;
-
-public class ServerStatusWatcher implements Runnable {
-
-
- private static final Logger LOG = LoggerFactory.getLogger(ServerStatusWatcher.class);
- private final ServerStatus serverStatus;
- private final InputStream fromParent;
- private final TikaServerConfig tikaServerConfig;
- private final Path forkedStatusPath;
- private final ByteBuffer statusBuffer = ByteBuffer.allocate(16);
- private volatile boolean shuttingDown = false;
-
- public ServerStatusWatcher(ServerStatus serverStatus, InputStream inputStream, Path forkedStatusPath, TikaServerConfig tikaServerConfig) throws InterruptedException {
- this.serverStatus = serverStatus;
- this.tikaServerConfig = tikaServerConfig;
- this.forkedStatusPath = forkedStatusPath;
- serverStatus.setStatus(ServerStatus.STATUS.OPERATING);
- this.fromParent = inputStream;
- Thread statusWatcher = new Thread(new StatusWatcher());
- statusWatcher.setDaemon(true);
- statusWatcher.start();
- writeStatus(false);
- }
-
- @Override
- public void run() {
-
-
- try {
- //this should block forever until the parent dies
- int directive = fromParent.read();
- if (directive != -1) {
- LOG.debug("Read byte ({}) from forking process. Shouldn't have received anything", directive);
- }
- } catch (Exception e) {
- LOG.debug("Exception reading from parent", e);
- } finally {
- LOG.info("Forking process signalled I should shutdown.");
- serverStatus.setStatus(ServerStatus.STATUS.PARENT_EXCEPTION);
- shutdown(ServerStatus.STATUS.PARENT_EXCEPTION);
- }
-
- }
-
- private synchronized void writeStatus(boolean shuttingDown) throws InterruptedException {
- if (this.shuttingDown == true) {
- return;
- }
- if (shuttingDown == true) {
- this.shuttingDown = true;
- }
-
- Instant started = Instant.now();
- long elapsed = Duration
- .between(started, Instant.now())
- .toMillis();
- try (FileChannel channel = FileChannel.open(forkedStatusPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
- while (elapsed < tikaServerConfig.getTaskTimeoutMillis()) {
- try (FileLock lock = channel.tryLock()) {
- if (lock != null) {
- ((Buffer) statusBuffer).position(0);
- statusBuffer.putLong(0, Instant
- .now()
- .toEpochMilli());
- statusBuffer.putInt(8, serverStatus
- .getStatus()
- .getInt());
- statusBuffer.putInt(12, serverStatus
- .getTasks()
- .size());
- channel.write(statusBuffer);
- channel.force(true);
- return;
- }
- } catch (IOException e) {
- LOG.warn("Problem writing to status file", e);
- }
- Thread.sleep(100);
- elapsed = Duration
- .between(started, Instant.now())
- .toMillis();
- }
- } catch (IOException e) {
- LOG.warn("Couldn't open forked status file for writing", e);
- }
- throw new FatalException("Couldn't write to status file after trying for " + elapsed + " millis.");
- }
-
- private void checkForHitMaxFiles() {
- if (tikaServerConfig.getMaxFiles() < 0) {
- return;
- }
- long filesProcessed = serverStatus.getFilesProcessed();
- if (filesProcessed >= tikaServerConfig.getMaxFiles()) {
- serverStatus.setStatus(ServerStatus.STATUS.HIT_MAX_FILES);
- }
- }
-
- private void checkForTaskTimeouts() {
- Instant now = Instant.now();
- for (TaskStatus status : serverStatus
- .getTasks()
- .values()) {
- long millisElapsed = Duration
- .between(status.started, now)
- .toMillis();
- if (millisElapsed > status.timeoutMillis) {
- serverStatus.setStatus(ServerStatus.STATUS.TIMEOUT);
- if (status.fileName.isPresent()) {
- LOG.error("Timeout task {}, millis elapsed {}, timeoutMillis {}, file id {}" + "consider increasing the allowable time with the " +
- "<taskTimeoutMillis/> parameter or the {} header", status.task.toString(), millisElapsed, status.timeoutMillis, status.fileName.get(),
- TimeoutConfig.X_TIKA_TIMEOUT_MILLIS);
- } else {
- LOG.error("Timeout task {}, millis elapsed {}; " + "consider increasing the allowable time with the " + "<taskTimeoutMillis/> parameter or the {} header",
- status.task.toString(), millisElapsed, TimeoutConfig.X_TIKA_TIMEOUT_MILLIS);
- }
- }
- }
- }
-
- private void shutdown(ServerStatus.STATUS status) {
- //if something went wrong with the parent,
- //the forked process should try to delete the tmp file
- if (status == ServerStatus.STATUS.PARENT_EXCEPTION) {
- try {
- Files.delete(forkedStatusPath);
- } catch (IOException e) {
- //swallow
- }
- } else {
- try {
- writeStatus(true);
- } catch (Exception e) {
- LOG.debug("problem writing status before shutdown", e);
- }
- }
-
- LOG.info("Shutting down forked process with status: {}", status.name());
- System.exit(status.getShutdownCode());
- }
-
- private static class FatalException extends RuntimeException {
- public FatalException() {
- super();
- }
-
- public FatalException(String msg) {
- super(msg);
- }
- }
-
- //This is an internal thread that pulses every ServerTimeouts#pingPulseMillis
- //within the forked process to see if the forked process should terminate.
- private class StatusWatcher implements Runnable {
-
- @Override
- public void run() {
- Instant lastWrite = Instant.now();
- while (true) {
- checkForHitMaxFiles();
- checkForTaskTimeouts();
- ServerStatus.STATUS currStatus = serverStatus.getStatus();
- if (currStatus != ServerStatus.STATUS.OPERATING) {
- LOG.warn("forked process observed " + currStatus.name() + " and is shutting down.");
- shutdown(currStatus);
- } else {
- long elapsed = Duration
- .between(lastWrite, Instant.now())
- .toMillis();
- if (elapsed > tikaServerConfig.getTaskPulseMillis()) {
- try {
- writeStatus(false);
- lastWrite = Instant.now();
- } catch (InterruptedException e) {
- LOG.debug("interrupted", e);
- }
- }
- }
-
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- LOG.warn("status watcher sees interrupted exception");
- return;
- }
- }
- }
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaLoggingFilter.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaLoggingFilter.java
deleted file mode 100644
index c07e8410b..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaLoggingFilter.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core;
-
-import java.io.IOException;
-
-import jakarta.ws.rs.container.ContainerRequestContext;
-import jakarta.ws.rs.container.ContainerRequestFilter;
-import jakarta.ws.rs.container.PreMatching;
-import jakarta.ws.rs.ext.Provider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@Provider
-@PreMatching
-public class TikaLoggingFilter implements ContainerRequestFilter {
- private static final Logger LOG = LoggerFactory.getLogger(TikaLoggingFilter.class);
-
- private boolean infoLevel;
-
- public TikaLoggingFilter(boolean infoLevel) {
- this.infoLevel = infoLevel;
- }
-
- @Override
- public void filter(ContainerRequestContext requestContext) throws IOException {
- String requestUri = requestContext
- .getUriInfo()
- .getRequestUri()
- .toString();
- if (infoLevel) {
- LOG.info("Request URI: {}", requestUri);
- } else {
- LOG.debug("Request URI: {}", requestUri);
- }
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
deleted file mode 100644
index b09f6af21..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core;
-
-import static org.apache.tika.server.core.TikaServerConfig.DEFAULT_HOST;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.help.HelpFormatter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TikaServerCli {
-
- private static final Logger LOG = LoggerFactory.getLogger(TikaServerCli.class);
- /**
- * This value is set to the server's id in the forked process.
- */
- public static String TIKA_SERVER_ID_ENV = "tika.server.id";
- private static List<TikaServerWatchDog> WATCHERS = new ArrayList<>();
-
- private static boolean PREVENT_STOP = false;
-
- private static Options getOptions() {
- Options options = new Options();
- options.addOption("h", "host", true, "host name (default = " + DEFAULT_HOST + ", use * for all)");
- options.addOption("p", "port", true,
- "listen port(s) (default = 9998)\n" + "Can specify multiple ports with inclusive ranges (e.g. 9990-9999)\n" + "or with comma delimited list (e.g. 9996,9998,9995)");
- options.addOption("?", "help", false, "this help message");
- options.addOption("c", "config", true, "tika-config file");
-
- options.addOption("i", "id", true, "id to use for server in" + " the server status endpoint and logging");
- options.addOption("noFork", "noFork", false, "runs in legacy 1.x mode -- " + "server runs in process and is not safely isolated in a forked process");
-
- return options;
- }
-
- public static void main(String[] args) {
- try {
- Options options = getOptions();
-
- CommandLineParser cliParser = new DefaultParser();
-
- CommandLine line = cliParser.parse(options, args);
- if (line.hasOption("help")) {
- usage(options);
- }
- TikaServerConfig tikaServerConfig = TikaServerConfig.load(line);
- PREVENT_STOP = tikaServerConfig.isPreventStopMethod();
-
- execute(tikaServerConfig);
- } catch (Exception e) {
- LOG.error("Can't start: ", e);
- System.exit(-1);
- }
- }
-
- private static void execute(TikaServerConfig tikaServerConfig) throws Exception {
-
- if (tikaServerConfig.isNoFork()) {
- noFork(tikaServerConfig);
- } else {
- try {
- mainLoop(tikaServerConfig);
- } catch (InterruptedException e) {
- //swallow
- LOG.debug("interrupted", e);
- }
- }
- }
-
- private static void mainLoop(TikaServerConfig tikaServerConfig) throws Exception {
-
- List<PortIdPair> portIdPairs = getPortIdPairs(tikaServerConfig);
-
- ExecutorService executorService = Executors.newFixedThreadPool(portIdPairs.size());
- ExecutorCompletionService<WatchDogResult> executorCompletionService = new ExecutorCompletionService<>(executorService);
-
- for (PortIdPair p : portIdPairs) {
- TikaServerWatchDog watcher = new TikaServerWatchDog(p.port, p.id, tikaServerConfig);
- executorCompletionService.submit(watcher);
- WATCHERS.add(watcher);
- }
-
- int finished = 0;
- try {
- while (finished < portIdPairs.size()) {
- Future<WatchDogResult> future = executorCompletionService.poll(1, TimeUnit.MINUTES);
- if (future != null) {
- LOG.debug("main loop future is available");
- WatchDogResult result = future.get();
- LOG.debug("main loop future: ({}); finished", result);
- finished++;
- }
- }
- } catch (InterruptedException e) {
- for (TikaServerWatchDog w : WATCHERS) {
- w.shutDown();
- }
- LOG.debug("thread interrupted", e);
- } finally {
- //this is just asking nicely...there is no guarantee!
- executorService.shutdownNow();
- }
- }
-
- public static void stop(String[] args) {
- if (PREVENT_STOP) {
- LOG.info("preventStopMethod was set to true in the server config. I'm not stopping.");
- }
- // process service stop function
- try {
- Options options = getStopOptions();
- CommandLineParser cliParser = new DefaultParser();
- CommandLine line = cliParser.parse(options, args);
- LOG.debug("Seeing 'preventSystemExit' on stop's commandline; not exiting");
- if (line.hasOption("preventSystemExit")) {
- return;
- }
- } catch (org.apache.commons.cli.ParseException e) {
- LOG.error("Can't parse stop arguments: ", e);
- System.exit(-1);
- }
-
- for (TikaServerWatchDog watcher : WATCHERS) {
- try {
- watcher.close();
- } catch (Exception e) {
- LOG.warn("Exception trying to close watcher", e);
- }
- }
- System.exit(0);
- }
-
- private static Options getStopOptions() {
- Options options = new Options();
- options.addOption("preventSystemExit", false,
- "Prevent the stop method from calling system.exit, " + "which would terminate the JVM. This is useful for integration tests.");
- return options;
- }
-
- private static String[] stripForkedArgs(String[] args) {
- List<String> ret = new ArrayList<>();
- for (String arg : args) {
- if (!arg.startsWith("-J")) {
- ret.add(arg);
- }
- }
- return ret.toArray(new String[0]);
- }
-
- public static void noFork(TikaServerConfig tikaServerConfig) throws Exception {
- List<String> args = tikaServerConfig.getForkedProcessArgs(tikaServerConfig.getPort(), tikaServerConfig.getIdBase());
- args.add("--noFork");
- TikaServerProcess.main(args.toArray(new String[0]));
- }
-
- private static void usage(Options options) throws IOException {
- HelpFormatter helpFormatter = HelpFormatter.builder().get();
- helpFormatter.printHelp("tikaserver", null, options, null, true);
- System.exit(-1);
- }
-
- private static List<PortIdPair> getPortIdPairs(TikaServerConfig tikaServerConfig) {
- List<PortIdPair> pairs = new ArrayList<>();
- int[] ports = tikaServerConfig.getPorts();
- //if there's only one port, use only the idbase, otherwise append -$port
- if (ports.length == 0) {
- throw new IllegalArgumentException("Couldn't find any ports in: " + tikaServerConfig.getPort());
- } else if (ports.length == 1) {
- pairs.add(new PortIdPair(ports[0], tikaServerConfig.getIdBase()));
- } else {
- for (int p : ports) {
- pairs.add(new PortIdPair(p, tikaServerConfig.getIdBase() + "-" + p));
- }
- }
-
- return pairs;
- }
-
-
- private static class PortIdPair {
- int port;
- String id;
-
- public PortIdPair(int port, String id) {
- this.port = port;
- this.id = id;
- }
- }
-
- /**
- * these are parameters that should not go
- * directly into the forked process. They
- * are either used by the forking process or
- * they are modified or may be modified before
- * creating the forked process.
- */
- private static class NonForkedValues {
- String portString;
- String id;
- int maxRestarts = -1;
-
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerConfig.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerConfig.java
deleted file mode 100644
index 663e95beb..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerConfig.java
+++ /dev/null
@@ -1,666 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.cli.CommandLine;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.config.ConfigBase;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.utils.ProcessUtils;
-import org.apache.tika.utils.StringUtils;
-import org.apache.tika.utils.XMLReaderUtils;
-
-public class TikaServerConfig extends ConfigBase {
-
- public static final int DEFAULT_PORT = 9998;
- public static final String DEFAULT_HOST = "localhost";
- public static final Set<String> LOG_LEVELS = new HashSet<>(Arrays.asList("debug", "info"));
- /**
- * Number of milliseconds to wait per server task (parse, detect, unpack, translate,
- * etc.) before timing out and shutting down the forked process.
- */
- public static final long DEFAULT_TASK_TIMEOUT_MILLIS = 300000;
- /**
- * Clients may not set a timeout less than this amount. This hinders
- * malicious clients from setting the timeout to a very low value
- * and DoS the server by forcing timeout restarts. Making tika-server
- * available to untrusted clients is dangerous.
- */
- public static final long DEFAULT_MINIMUM_TIMEOUT_MILLIS = 30000;
- /**
- * How often to check to see that the task hasn't timed out
- */
- public static final long DEFAULT_TASK_PULSE_MILLIS = 10000;
- /**
- * Number of milliseconds to wait for forked process to startup
- */
- public static final long DEFAULT_FORKED_STARTUP_MILLIS = 120000;
- private static final Logger LOG = LoggerFactory.getLogger(TikaServerConfig.class);
- //used in fork mode -- restart after processing this many files
- private static final long DEFAULT_MAX_FILES = 100000;
- private static final int DEFAULT_DIGEST_MARK_LIMIT = 20 * 1024 * 1024;
- private static final String UNSECURE_WARNING =
- "WARNING: You have chosen to run tika-server with unsecure features enabled.\n" + "Whoever has access to your service now has the same read permissions\n" +
- "as you've given your fetchers and the same write permissions " + "as your emitters.\n" + "Users could request and receive a sensitive file from your\n" +
- "drive or a webpage from your intranet and/or send malicious content to\n" + " your emitter endpoints. See CVE-2015-3271.\n" +
- "Please make sure you know what you are doing.";
- private static final List<String> ONLY_IN_FORK_MODE = Arrays.asList(
- new String[]{"taskTimeoutMillis", "taskPulseMillis", "maxFiles", "javaPath", "maxRestarts", "numRestarts", "forkedStatusFile", "maxForkedStartupMillis",
- "tmpFilePrefix"});
- private static Pattern SYS_PROPS = Pattern.compile("\\$\\{sys:([-_0-9A-Za-z]+)\\}");
- /*
-TODO: integrate these settings:
- * Number of milliseconds to wait to start forked process.
-public static final long DEFAULT_FORKED_PROCESS_STARTUP_MILLIS = 60000;
-
- * Maximum number of milliseconds to wait to shutdown forked process to allow
- * for current parses to complete.
-public static final long DEFAULT_FORKED_PROCESS_SHUTDOWN_MILLIS = 30000;
-
-private long forkedProcessStartupMillis = DEFAULT_FORKED_PROCESS_STARTUP_MILLIS;
-
-private long forkedProcessShutdownMillis = DEFAULT_FORKED_PROCESS_SHUTDOWN_MILLIS;
-
- */
- private int maxRestarts = -1;
- private long maxFiles = 100000;
- private long taskTimeoutMillis = DEFAULT_TASK_TIMEOUT_MILLIS;
- private long minimumTimeoutMillis = DEFAULT_MINIMUM_TIMEOUT_MILLIS;
- private long taskPulseMillis = DEFAULT_TASK_PULSE_MILLIS;
- private long maxforkedStartupMillis = DEFAULT_FORKED_STARTUP_MILLIS;
- private boolean enableUnsecureFeatures = false;
- private String cors = "";
- private boolean returnStackTrace = false;
- private boolean noFork = false;
- //TODO: make parameterizable for debugging
- private String tempFilePrefix = "apache-tika-server-forked-tmp-";
- private Set<String> supportedFetchers = new HashSet<>();
- private Set<String> supportedEmitters = new HashSet<>();
- private List<String> forkedJvmArgs = new ArrayList<>();
- private String idBase = UUID
- .randomUUID()
- .toString();
- private String port = Integer.toString(DEFAULT_PORT);
- private String host = DEFAULT_HOST;
- private int digestMarkLimit = DEFAULT_DIGEST_MARK_LIMIT;
- private String digest = "";
- private String javaPath = "java";
- //debug or info only
- private String logLevel = "";
- private Path configPath;
- private List<String> endpoints = new ArrayList<>();
- //these should only be set in the forked process
- //and they are automatically set by the forking process
- private String forkedStatusFile;
- private int numRestarts = 0;
-
- private boolean preventStopMethod = false;
-
- private TlsConfig tlsConfig = new TlsConfig();
-
- /**
- * Config with only the defaults
- */
- public static TikaServerConfig load() {
- return new TikaServerConfig();
- }
-
- public static TikaServerConfig load(CommandLine commandLine) throws IOException, TikaException {
-
- TikaServerConfig config = null;
- Set<String> settings = new HashSet<>();
- if (commandLine.hasOption("c")) {
- config = load(Paths.get(commandLine.getOptionValue("c")), commandLine, settings);
- } else {
- config = new TikaServerConfig();
- }
-
- //port, host, nofork and id can be overwritten on the commandline at runtime
- if (commandLine.hasOption("p")) {
- config.setPort(commandLine.getOptionValue("p"));
- settings.add("port");
- }
-
- if (commandLine.hasOption("h")) {
- config.setHost(commandLine.getOptionValue("h"));
- settings.add("host");
- }
-
- if (commandLine.hasOption("noFork")) {
- config.setNoFork(true);
- settings.add("noFork");
- }
-
- if (commandLine.hasOption("i")) {
- config.setId(commandLine.getOptionValue("i"));
- settings.add("id");
- }
-
- //this should only be set by the parent process
- if (commandLine.hasOption("numRestarts")) {
- config.setNumRestarts(Integer.parseInt(commandLine.getOptionValue("numRestarts")));
- settings.add("numRestarts");
- }
-
- if (commandLine.hasOption("forkedStatusFile")) {
- config.setForkedStatusFile(commandLine.getOptionValue("forkedStatusFile"));
- settings.add("forkedStatusFile");
- }
- config.validateConsistency(settings);
- return config;
- }
-
- static TikaServerConfig load(Path p, CommandLine commandLine, Set<String> settings) throws IOException, TikaException {
- try (InputStream is = Files.newInputStream(p)) {
- TikaServerConfig config = TikaServerConfig.load(is, commandLine, settings);
- if (config.getConfigPath() == null) {
- config.setConfigPath(p
- .toAbsolutePath()
- .toString());
- }
- loadSupportedFetchersEmitters(config);
- return config;
- }
- }
-
- private static TikaServerConfig load(InputStream is, CommandLine commandLine, Set<String> settings) throws IOException, TikaException {
- TikaServerConfig tikaServerConfig = new TikaServerConfig();
- Set<String> configSettings = tikaServerConfig.configure("server", is);
- settings.addAll(configSettings);
- //override a few things in config file
- if (commandLine.hasOption("noFork")) {
- tikaServerConfig.setNoFork(true);
- }
- return tikaServerConfig;
- }
-
- private static void loadSupportedFetchersEmitters(TikaServerConfig tikaServerConfig) throws IOException, TikaConfigException {
- //this is an abomination... clean up this double read
- try (InputStream is = Files.newInputStream(tikaServerConfig.getConfigPath())) {
- Node properties = null;
- try {
- properties = XMLReaderUtils
- .buildDOM(is)
- .getDocumentElement();
- } catch (SAXException e) {
- throw new IOException(e);
- } catch (TikaException e) {
- throw new TikaConfigException("problem loading xml to dom", e);
- }
- if (!properties
- .getLocalName()
- .equals("properties")) {
- throw new TikaConfigException("expect properties as root node");
- }
- NodeList children = properties.getChildNodes();
- for (int i = 0; i < children.getLength(); i++) {
- Node child = children.item(i);
- if ("fetchers".equals(child.getLocalName())) {
- loadSupported(child, "fetcher", tikaServerConfig.supportedFetchers);
- } else if ("emitters".equals(child.getLocalName())) {
- loadSupported(child, "emitter", tikaServerConfig.supportedEmitters);
- }
- }
- }
- }
-
- private static void loadSupported(Node compound, String itemName, Set<String> supported) {
- NodeList children = compound.getChildNodes();
- for (int i = 0; i < children.getLength(); i++) {
- Node child = children.item(i);
- if (itemName.equals(child.getLocalName())) {
- String name = getName(child);
- if (name != null) {
- supported.add(name);
- }
- }
- }
- }
-
- private static String getName(Node fetcherOrEmitter) {
- NodeList children = fetcherOrEmitter.getChildNodes();
- for (int i = 0; i < children.getLength(); i++) {
- Node child = children.item(i);
- if ("params".equals(child.getLocalName())) {
- NodeList params = child.getChildNodes();
- for (int j = 0; j < params.getLength(); j++) {
- Node param = params.item(j);
- if ("name".equals(param.getLocalName())) {
- return param.getTextContent();
- }
- }
- } else if ("name".equals(child.getLocalName())) {
- return child.getTextContent();
- }
- }
- return null;
- }
-
- protected static List<String> interpolateSysProps(List<String> forkedJvmArgs) {
- List<String> ret = new ArrayList<>();
- for (String arg : forkedJvmArgs) {
- if (arg.startsWith("-D")) {
- String interpolated = interpolate(arg);
- ret.add(interpolated);
- } else {
- ret.add(arg);
- }
- }
- return ret;
- }
-
- private static String interpolate(String arg) {
- StringBuilder sb = new StringBuilder();
- Matcher m = SYS_PROPS.matcher(arg);
- while (m.find()) {
- String prop = System.getProperty(m.group(1));
- LOG.debug("interpolating {} -> {}", m.group(1), prop);
- if (prop == null) {
- LOG.warn("no system property set for {}, falling back to {}", m.group(1), arg);
- return arg;
- }
- m.appendReplacement(sb, Matcher.quoteReplacement(prop));
- }
- m.appendTail(sb);
- return sb.toString();
- }
-
- public boolean isNoFork() {
- return noFork;
- }
-
- public void setNoFork(boolean noFork) {
- this.noFork = noFork;
- }
-
- public String getPort() {
- return port;
- }
-
- public void setPort(String port) {
- this.port = port;
- }
-
- /**
- * How long to wait for a task before shutting down the forked server process
- * and restarting it.
- *
- * @return
- */
- public long getTaskTimeoutMillis() {
- return taskTimeoutMillis;
- }
-
- /**
- * @param taskTimeoutMillis number of milliseconds to allow per task
- * (parse, detection, unzipping, etc.)
- */
- public void setTaskTimeoutMillis(long taskTimeoutMillis) {
- this.taskTimeoutMillis = taskTimeoutMillis;
- }
-
- /**
- * How often to check to see that a task has timed out
- *
- * @return
- */
- public long getTaskPulseMillis() {
- return taskPulseMillis;
- }
-
- public void setTaskPulseMillis(long taskPulseMillis) {
- this.taskPulseMillis = taskPulseMillis;
- }
-
- public int getMaxRestarts() {
- return maxRestarts;
- }
-
- public void setMaxRestarts(int maxRestarts) {
- this.maxRestarts = maxRestarts;
- }
-
- /**
- * Maximum time in millis to allow for the forked process to startup
- * or restart
- *
- * @return
- */
- public long getMaxForkedStartupMillis() {
- return maxforkedStartupMillis;
- }
-
- public void setMaxForkedStartupMillis(long maxForkedStartupMillis) {
- this.maxforkedStartupMillis = maxForkedStartupMillis;
- }
-
- public List<String> getForkedProcessArgs(String portString, String id) {
- int[] ports = getPorts(portString);
- if (ports.length > 1 || ports.length == 0) {
- throw new IllegalArgumentException("must specify one and only one port here:" + portString);
- }
- int port = ports[0];
- return getForkedProcessArgs(port, id);
- }
-
- public List<String> getForkedProcessArgs(int port, String id) {
- //these are the arguments for the forked process
- List<String> args = new ArrayList<>();
- args.add("-h");
- args.add(getHost());
- args.add("-p");
- args.add(Integer.toString(port));
- args.add("-i");
- args.add(id);
- if (hasConfigFile()) {
- args.add("-c");
- args.add(ProcessUtils.escapeCommandLine(configPath
- .toAbsolutePath()
- .toString()));
- }
- return args;
- }
-
- public long getMinimumTimeoutMillis() {
- return minimumTimeoutMillis;
- }
-
- public void setMinimumTimeoutMillis(long minimumTimeoutMillis) {
- this.minimumTimeoutMillis = minimumTimeoutMillis;
- }
-
- public String getIdBase() {
- return idBase;
- }
-
- /**
- * full path to the java executable
- *
- * @return
- */
- public String getJavaPath() {
- return javaPath;
- }
-
- public void setJavaPath(String javaPath) {
- this.javaPath = javaPath;
- }
-
- public List<String> getForkedJvmArgs() {
- //defensively copy
- return new ArrayList<>(forkedJvmArgs);
- }
-
- public void setForkedJvmArgs(List<String> forkedJvmArgs) {
- this.forkedJvmArgs = new ArrayList<>(interpolateSysProps(forkedJvmArgs));
- }
-
- public String getTempFilePrefix() {
- return tempFilePrefix;
- }
-
- public boolean isEnableUnsecureFeatures() {
- return enableUnsecureFeatures;
- }
-
- public void setEnableUnsecureFeatures(boolean enableUnsecureFeatures) {
- this.enableUnsecureFeatures = enableUnsecureFeatures;
- }
-
- private void validateConsistency(Set<String> settings) throws TikaConfigException {
- if (host == null) {
- throw new TikaConfigException("Must specify 'host'");
- }
-
- if (!StringUtils.isBlank(port)) {
- setPort(port);
- }
-
- if (isNoFork()) {
- for (String onlyFork : ONLY_IN_FORK_MODE) {
- if (settings.contains(onlyFork)) {
- throw new TikaConfigException("Can't set param=" + onlyFork + "if you've selected noFork");
- }
- }
- }
- //add headless if not already configured
- boolean foundHeadlessOption = forkedJvmArgs
- .stream()
- .anyMatch(arg -> arg.contains("java.awt.headless"));
- //if user has already specified headless...don't modify
- if (!foundHeadlessOption) {
- forkedJvmArgs.add("-Djava.awt.headless=true");
- }
-
-
- }
-
- public String getHost() {
- return host;
- }
-
- public void setHost(String host) {
- if ("*".equals(host)) {
- host = "0.0.0.0";
- }
- this.host = host;
- }
-
- public String getLogLevel() {
- return logLevel;
- }
-
- public void setLogLevel(String level) throws TikaConfigException {
- if (level.equals("debug") || level.equals("info")) {
- this.logLevel = level;
- } else {
- throw new TikaConfigException("log level must be one of: 'debug' or 'info'");
- }
- }
-
- /**
- * @return the origin url for cors, can be "*"
- */
- public String getCors() {
- return cors;
- }
-
- public void setCors(String cors) {
- this.cors = cors;
- }
-
- public boolean hasConfigFile() {
- return configPath != null;
- }
-
- public Path getConfigPath() {
- return configPath;
- }
-
- public void setConfigPath(String path) {
- this.configPath = Paths.get(path);
- }
-
- public int getDigestMarkLimit() {
- return digestMarkLimit;
- }
-
- public void setDigestMarkLimit(int digestMarkLimit) {
- this.digestMarkLimit = digestMarkLimit;
- }
-
- /**
- * digest configuration string, e.g. md5 or sha256, alternately w 16 or 32 encoding,
- * e.g. md5:32,sha256:16 would result in two digests per file
- *
- * @return
- */
- public String getDigest() {
- return digest;
- }
-
- public void setDigest(String digest) {
- LOG.info("As of Tika 2.5.0, you can set the digester via the AutoDetectParserConfig in " + "tika-config.xml. We plan to remove this commandline option in 2.8.0");
- this.digest = digest;
- }
-
- /**
- * maximum number of files before the forked server restarts.
- * This is useful for avoiding any slow-building memory leaks/bloat.
- *
- * @return
- */
- public long getMaxFiles() {
- return maxFiles;
- }
-
- public void setMaxFiles(long maxFiles) {
- this.maxFiles = maxFiles;
- }
-
- public boolean isReturnStackTrace() {
- return returnStackTrace;
- }
-
- public void setReturnStackTrace(boolean returnStackTrace) {
- this.returnStackTrace = returnStackTrace;
- }
-
- public TlsConfig getTlsConfig() {
- return tlsConfig;
- }
-
- public void setTlsConfig(TlsConfig tlsConfig) {
- this.tlsConfig = tlsConfig;
- }
-
- public List<String> getEndpoints() {
- return endpoints;
- }
-
- public void setEndpoints(List<String> endpoints) {
- this.endpoints = new ArrayList<>(endpoints);
- }
-
- public String getId() {
- //TODO fix this
- return idBase;
- }
-
- public void setId(String id) {
- this.idBase = id;
- }
-
- private void addEndPoints(List<String> endPoints) {
- this.endpoints.addAll(endPoints);
- }
-
- private void addJVMArgs(List<String> args) {
- this.forkedJvmArgs.addAll(args);
- }
-
- public int getNumRestarts() {
- return numRestarts;
- }
-
- /******
- * these should only be used in the commandline for a forked process
- ******/
-
-
- private void setNumRestarts(int numRestarts) {
- this.numRestarts = numRestarts;
- }
-
- public String getForkedStatusFile() {
- return forkedStatusFile;
- }
-
- private void setForkedStatusFile(String forkedStatusFile) {
- this.forkedStatusFile = forkedStatusFile;
- }
-
- public void setMaxforkedStartupMillis(long maxforkedStartupMillis) {
- this.maxforkedStartupMillis = maxforkedStartupMillis;
- }
-
- public boolean isPreventStopMethod() {
- return preventStopMethod;
- }
-
- public void setPreventStopMethod(boolean preventStopMethod) {
- this.preventStopMethod = preventStopMethod;
- }
-
- public int[] getPorts() {
- return getPorts(port);
- }
-
- private int[] getPorts(String portString) {
- //throws NumberFormatException
- Matcher rangeMatcher = Pattern
- .compile("^(\\d+)-(\\d+)\\Z")
- .matcher("");
- String[] commaDelimited = portString.split(",");
- List<Integer> indivPorts = new ArrayList<>();
- for (String val : commaDelimited) {
- rangeMatcher.reset(val);
- if (rangeMatcher.find()) {
- int min = Math.min(Integer.parseInt(rangeMatcher.group(1)), Integer.parseInt(rangeMatcher.group(2)));
- int max = Math.max(Integer.parseInt(rangeMatcher.group(1)), Integer.parseInt(rangeMatcher.group(2)));
- for (int i = min; i <= max; i++) {
- indivPorts.add(i);
- }
- } else {
- indivPorts.add(Integer.parseInt(val));
- }
- }
- return indivPorts
- .stream()
- .mapToInt(Integer::intValue)
- .toArray();
- }
-
- public Set<String> getSupportedFetchers() {
- return supportedFetchers;
- }
-
- public Set<String> getSupportedEmitters() {
- return supportedEmitters;
- }
-
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerParseExceptionMapper.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerParseExceptionMapper.java
deleted file mode 100644
index cbf5bfbd1..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerParseExceptionMapper.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core;
-
-
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.io.Writer;
-
-import jakarta.ws.rs.WebApplicationException;
-import jakarta.ws.rs.core.Response;
-import jakarta.ws.rs.ext.ExceptionMapper;
-import jakarta.ws.rs.ext.Provider;
-
-import org.apache.tika.exception.EncryptedDocumentException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.exception.UnsupportedFormatException;
-
-@Provider
-public class TikaServerParseExceptionMapper implements ExceptionMapper<TikaServerParseException> {
-
- private final boolean returnStack;
-
- public TikaServerParseExceptionMapper(boolean returnStack) {
- this.returnStack = returnStack;
- }
-
- public Response toResponse(TikaServerParseException e) {
- if (e.getMessage() != null && e
- .getMessage()
- .equals(Response.Status.UNSUPPORTED_MEDIA_TYPE.toString())) {
- return buildResponse(e, 415);
- }
- Throwable cause = e.getCause();
- if (cause == null) {
- return buildResponse(e, Response.Status.INTERNAL_SERVER_ERROR.getStatusCode());
- } else {
- if (cause instanceof EncryptedDocumentException) {
- return buildResponse(cause, 422);
- } else if (cause instanceof TikaException) {
- //unsupported media type
- Throwable causeOfCause = cause.getCause();
- if (causeOfCause instanceof WebApplicationException) {
- return ((WebApplicationException) causeOfCause).getResponse();
- }
- return buildResponse(cause, 422);
- } else if (cause instanceof IllegalStateException) {
- return buildResponse(cause, 422);
- } else if (cause instanceof UnsupportedFormatException) {
- return buildResponse(cause, 422);
- } else if (cause instanceof WebApplicationException) {
- return ((WebApplicationException) e.getCause()).getResponse();
- } else {
- return buildResponse(e, 500);
- }
- }
- }
-
- private Response buildResponse(Throwable cause, int i) {
- if (returnStack && cause != null) {
- Writer result = new StringWriter();
- PrintWriter writer = new PrintWriter(result);
- cause.printStackTrace(writer);
- writer.flush();
- try {
- result.flush();
- } catch (IOException e) {
- //something went seriously wrong
- return Response
- .status(500)
- .build();
- }
- return Response
- .status(i)
- .entity(result.toString())
- .type("text/plain")
- .build();
- } else {
- return Response
- .status(i)
- .build();
- }
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
deleted file mode 100644
index e1afe2491..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
+++ /dev/null
@@ -1,514 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.BindException;
-import java.nio.file.Paths;
-import java.security.GeneralSecurityException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.Options;
-import org.apache.cxf.binding.BindingFactoryManager;
-import org.apache.cxf.configuration.jsse.TLSParameterJaxBUtils;
-import org.apache.cxf.configuration.jsse.TLSServerParameters;
-import org.apache.cxf.configuration.security.ClientAuthentication;
-import org.apache.cxf.configuration.security.KeyManagersType;
-import org.apache.cxf.configuration.security.KeyStoreType;
-import org.apache.cxf.configuration.security.TrustManagersType;
-import org.apache.cxf.endpoint.Server;
-import org.apache.cxf.jaxrs.JAXRSBindingFactory;
-import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
-import org.apache.cxf.jaxrs.lifecycle.ResourceProvider;
-import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
-import org.apache.cxf.jaxrs.utils.JAXRSServerFactoryCustomizationUtils;
-import org.apache.cxf.rs.security.cors.CrossOriginResourceSharingFilter;
-import org.apache.cxf.service.factory.ServiceConstructionException;
-import org.apache.cxf.transport.common.gzip.GZIPInInterceptor;
-import org.apache.cxf.transport.common.gzip.GZIPOutInterceptor;
-import org.apache.cxf.transport.http_jetty.JettyHTTPServerEngineFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.Tika;
-import org.apache.tika.config.ServiceLoader;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.parser.DigestingParser;
-import org.apache.tika.parser.digestutils.BouncyCastleDigester;
-import org.apache.tika.parser.digestutils.CommonsDigester;
-import org.apache.tika.pipes.core.emitter.EmitterManager;
-import org.apache.tika.pipes.core.fetcher.FetcherManager;
-import org.apache.tika.server.core.resource.AsyncResource;
-import org.apache.tika.server.core.resource.DetectorResource;
-import org.apache.tika.server.core.resource.LanguageResource;
-import org.apache.tika.server.core.resource.MetadataResource;
-import org.apache.tika.server.core.resource.PipesResource;
-import org.apache.tika.server.core.resource.RecursiveMetadataResource;
-import org.apache.tika.server.core.resource.TikaDetectors;
-import org.apache.tika.server.core.resource.TikaMimeTypes;
-import org.apache.tika.server.core.resource.TikaParsers;
-import org.apache.tika.server.core.resource.TikaResource;
-import org.apache.tika.server.core.resource.TikaServerResource;
-import org.apache.tika.server.core.resource.TikaServerStatus;
-import org.apache.tika.server.core.resource.TikaVersion;
-import org.apache.tika.server.core.resource.TikaWelcome;
-import org.apache.tika.server.core.resource.TranslateResource;
-import org.apache.tika.server.core.resource.UnpackerResource;
-import org.apache.tika.server.core.writer.CSVMessageBodyWriter;
-import org.apache.tika.server.core.writer.JSONMessageBodyWriter;
-import org.apache.tika.server.core.writer.JSONObjWriter;
-import org.apache.tika.server.core.writer.MetadataListMessageBodyWriter;
-import org.apache.tika.server.core.writer.TarWriter;
-import org.apache.tika.server.core.writer.TextMessageBodyWriter;
-import org.apache.tika.server.core.writer.ZipWriter;
-import org.apache.tika.utils.StringUtils;
-
-public class TikaServerProcess {
-
-
- public static final Set<String> LOG_LEVELS = new HashSet<>(Arrays.asList("debug", "info"));
- public static final int BIND_EXCEPTION = 42;
- private static final Logger LOG = LoggerFactory.getLogger(TikaServerProcess.class);
- public static int DO_NOT_RESTART_EXIT_VALUE = -100;
-
- private static Options getOptions() {
- Options options = new Options();
- options.addOption("h", "host", true, "host name, use * for all)");
- options.addOption("p", "port", true, "listen port");
- options.addOption("c", "config", true, "Tika Configuration file to override default config with.");
- options.addOption("i", "id", true, "id to use for server in server status endpoint");
- options.addOption("?", "help", false, "this help message");
- options.addOption("noFork", "noFork", false, "if launched in no fork mode");
- options.addOption("forkedStatusFile", true,
- "Not allowed in -noFork: temporary file used to communicate " + "with forking process -- do not use this! " + "Should only be invoked by forking process.");
- options.addOption("tmpFilePrefix", true, "Not allowed in -noFork: prefix for temp file - for debugging only");
- options.addOption("numRestarts", true, "Not allowed in -noFork: number of times that " + "the forked server has had to be restarted.");
- return options;
- }
-
- public static void main(String[] args) throws Exception {
- LOG.info("Starting {} server", Tika.getString());
- try {
- Options options = getOptions();
- CommandLineParser cliParser = new DefaultParser();
- CommandLine line = cliParser.parse(options, args);
- TikaServerConfig tikaServerConfig = TikaServerConfig.load(line);
- LOG.debug("forked config: {}", tikaServerConfig);
-
- ServerDetails serverDetails = initServer(tikaServerConfig);
- startServer(serverDetails, tikaServerConfig);
-
- } catch (Exception e) {
- LOG.error("Can't start: ", e);
- System.exit(-1);
- }
- }
-
- private static boolean isBindException(Throwable e) {
- if (e == null) {
- return false;
- }
- if (e instanceof BindException) {
- return true;
- }
- return isBindException(e.getCause());
- }
-
- private static void startServer(ServerDetails serverDetails, TikaServerConfig tikaServerConfig) throws Exception {
-
- try {
- //start the server
- Server server = serverDetails.sf.create();
- } catch (ServiceConstructionException e) {
- LOG.warn("exception starting server", e);
- if (isBindException(e)) {
- System.exit(BIND_EXCEPTION);
- }
- System.exit(DO_NOT_RESTART_EXIT_VALUE);
- }
-
- if (!tikaServerConfig.isNoFork()) {
- //redirect
- InputStream in = System.in;
- System.setIn(new ByteArrayInputStream(new byte[0]));
-
- String forkedStatusFile = tikaServerConfig.getForkedStatusFile();
- Thread serverThread = new Thread(new ServerStatusWatcher(serverDetails.serverStatus, in, Paths.get(forkedStatusFile), tikaServerConfig));
-
- serverThread.start();
- }
-
- LOG.info("Started Apache Tika server {} at {}", serverDetails.serverId, serverDetails.url);
- }
-
- //This returns the server, configured and ready to be started.
- private static ServerDetails initServer(TikaServerConfig tikaServerConfig) throws Exception {
- String host = tikaServerConfig.getHost();
- int[] ports = tikaServerConfig.getPorts();
- if (ports.length > 1) {
- throw new IllegalArgumentException("there must be only one port here! " + "I see: " + tikaServerConfig.getPort());
- }
-
- int port = ports[0];
- // The Tika Configuration to use throughout
- TikaConfig tika;
-
- if (tikaServerConfig.hasConfigFile()) {
- LOG.info("Using custom config: {}", tikaServerConfig.getConfigPath());
- tika = new TikaConfig(tikaServerConfig.getConfigPath());
- } else {
- tika = TikaConfig.getDefaultConfig();
- }
-
- DigestingParser.Digester digester = null;
- if (!StringUtils.isBlank(tikaServerConfig.getDigest())) {
- try {
- digester = new CommonsDigester(tikaServerConfig.getDigestMarkLimit(), tikaServerConfig.getDigest());
- } catch (IllegalArgumentException commonsException) {
- try {
- digester = new BouncyCastleDigester(tikaServerConfig.getDigestMarkLimit(), tikaServerConfig.getDigest());
- } catch (IllegalArgumentException bcException) {
- throw new IllegalArgumentException(
- "Tried both CommonsDigester (" + commonsException.getMessage() + ") and BouncyCastleDigester (" + bcException.getMessage() + ")", bcException);
- }
- }
- }
-
- //TODO -- clean this up -- only load as necessary
- FetcherManager fetcherManager = null;
- InputStreamFactory inputStreamFactory = null;
- if (tikaServerConfig.isEnableUnsecureFeatures()) {
- fetcherManager = FetcherManager.load(tikaServerConfig.getConfigPath());
- inputStreamFactory = new FetcherStreamFactory(fetcherManager);
- } else {
- inputStreamFactory = new DefaultInputStreamFactory();
- }
- //TODO -- figure out how to turn this back on
- //logFetchersAndEmitters(tikaServerConfig.isEnableUnsecureFeatures(), fetcherManager,
- // emitterManager);
-
- String serverId = tikaServerConfig.getId();
- LOG.debug("SERVER ID:" + serverId);
- ServerStatus serverStatus;
-
- if (tikaServerConfig.isNoFork()) {
- serverStatus = new ServerStatus(serverId, 0, true);
- } else {
- serverStatus = new ServerStatus(serverId, tikaServerConfig.getNumRestarts(), false);
- System.setOut(System.err);
- }
- TikaResource.init(tika, tikaServerConfig, digester, inputStreamFactory, serverStatus);
- JAXRSServerFactoryBean sf = new JAXRSServerFactoryBean();
-
- List<ResourceProvider> resourceProviders = new ArrayList<>();
- List<Object> providers = new ArrayList<>();
- loadAllProviders(tikaServerConfig, serverStatus, resourceProviders, providers);
-
- sf.setResourceProviders(resourceProviders);
-
- sf.setProviders(providers);
-
- //set compression interceptors
- sf.setOutInterceptors(Collections.singletonList(new GZIPOutInterceptor()));
- sf.setInInterceptors(Collections.singletonList(new GZIPInInterceptor()));
-
- String protocol = tikaServerConfig
- .getTlsConfig()
- .isActive() ? "https" : "http";
- String url = protocol + "://" + host + ":" + port + "/";
- sf.setAddress(url);
- sf.setResourceComparator(new ProduceTypeResourceComparator());
- BindingFactoryManager manager = sf
- .getBus()
- .getExtension(BindingFactoryManager.class);
- if (tikaServerConfig
- .getTlsConfig()
- .isActive()) {
- LOG.warn("The TLS configuration is in BETA and might change " + "dramatically in future releases.");
- TLSServerParameters tlsParams = getTlsParams(tikaServerConfig.getTlsConfig());
- JettyHTTPServerEngineFactory factory = new JettyHTTPServerEngineFactory();
- factory.setBus(sf.getBus());
- factory.setTLSServerParametersForPort(host, port, tlsParams);
- JAXRSServerFactoryCustomizationUtils.customize(sf);
- } else {
- JAXRSBindingFactory factory = new JAXRSBindingFactory();
- factory.setBus(sf.getBus());
- manager.registerBindingFactory(JAXRSBindingFactory.JAXRS_BINDING_ID, factory);
- }
- ServerDetails details = new ServerDetails();
- details.sf = sf;
- details.url = url;
- details.serverId = serverId;
- details.serverStatus = serverStatus;
- return details;
- }
-
- private static TLSServerParameters getTlsParams(TlsConfig tlsConfig) throws GeneralSecurityException, IOException {
- KeyStoreType keyStore = new KeyStoreType();
- keyStore.setType(tlsConfig.getKeyStoreType());
- keyStore.setPassword(tlsConfig.getKeyStorePassword());
- keyStore.setFile(tlsConfig.getKeyStoreFile());
- KeyManagersType kmt = new KeyManagersType();
- kmt.setKeyStore(keyStore);
- kmt.setKeyPassword(tlsConfig.getKeyStorePassword());
- TLSServerParameters parameters = new TLSServerParameters();
- parameters.setKeyManagers(TLSParameterJaxBUtils.getKeyManagers(kmt));
-
- if (tlsConfig.hasTrustStore()) {
- KeyStoreType trustKeyStore = new KeyStoreType();
- trustKeyStore.setType(tlsConfig.getTrustStoreType());
- trustKeyStore.setPassword(tlsConfig.getTrustStorePassword());
- trustKeyStore.setFile(tlsConfig.getTrustStoreFile());
- TrustManagersType tmt = new TrustManagersType();
- tmt.setKeyStore(trustKeyStore);
- parameters.setTrustManagers(TLSParameterJaxBUtils.getTrustManagers(tmt, true));
- }
- ClientAuthentication clientAuthentication = new ClientAuthentication();
- clientAuthentication.setRequired(tlsConfig.isClientAuthenticationRequired());
- clientAuthentication.setWant(tlsConfig.isClientAuthenticationWanted());
- parameters.setClientAuthentication(clientAuthentication);
- return parameters;
- }
-
- private static void loadAllProviders(TikaServerConfig tikaServerConfig, ServerStatus serverStatus, List<ResourceProvider> resourceProviders, List<Object> writers)
- throws TikaException, SAXException, IOException {
- List<ResourceProvider> tmpCoreProviders = loadCoreProviders(tikaServerConfig, serverStatus);
-
- resourceProviders.addAll(tmpCoreProviders);
- resourceProviders.add(new SingletonResourceProvider(new TikaWelcome(tmpCoreProviders)));
-
- //for now, just load everything
- //TODO figure out which ones to turn off
- writers.add(new TarWriter());
- writers.add(new ZipWriter());
- writers.add(new CSVMessageBodyWriter());
- writers.add(new MetadataListMessageBodyWriter());
- writers.add(new JSONMessageBodyWriter());
- writers.add(new TextMessageBodyWriter());
- writers.addAll(loadWriterServices());
- writers.add(new TikaServerParseExceptionMapper(tikaServerConfig.isReturnStackTrace()));
- writers.add(new JSONObjWriter());
-
- TikaLoggingFilter logFilter = null;
- if (!StringUtils.isBlank(tikaServerConfig.getLogLevel())) {
- String logLevel = tikaServerConfig.getLogLevel();
- if (LOG_LEVELS.contains(logLevel)) {
- boolean isInfoLevel = "info".equals(logLevel);
- logFilter = new TikaLoggingFilter(isInfoLevel);
- writers.add(logFilter);
- } else {
- LOG.warn("Unsupported request URI log level: {}", logLevel);
- }
- }
-
- CrossOriginResourceSharingFilter corsFilter = null;
- if (!StringUtils.isBlank(tikaServerConfig.getCors())) {
- corsFilter = new CrossOriginResourceSharingFilter();
- String url = tikaServerConfig.getCors();
- List<String> origins = new ArrayList<>();
- if (!url.equals("*")) {
- origins.add(url); // Empty list allows all origins.
- }
- corsFilter.setAllowOrigins(origins);
- writers.add(corsFilter);
- }
-
- }
-
- private static List<ResourceProvider> loadCoreProviders(TikaServerConfig tikaServerConfig, ServerStatus serverStatus) throws TikaException, IOException, SAXException {
- List<ResourceProvider> resourceProviders = new ArrayList<>();
- boolean addAsyncResource = false;
- boolean addPipesResource = false;
- if (tikaServerConfig
- .getEndpoints()
- .size() == 0) {
- resourceProviders.add(new SingletonResourceProvider(new MetadataResource()));
- resourceProviders.add(new SingletonResourceProvider(new RecursiveMetadataResource()));
- resourceProviders.add(new SingletonResourceProvider(new DetectorResource(serverStatus)));
- resourceProviders.add(new SingletonResourceProvider(new LanguageResource()));
- resourceProviders.add(new SingletonResourceProvider(new TranslateResource(serverStatus, tikaServerConfig.getTaskTimeoutMillis())));
- resourceProviders.add(new SingletonResourceProvider(new TikaResource()));
- resourceProviders.add(new SingletonResourceProvider(new UnpackerResource()));
- resourceProviders.add(new SingletonResourceProvider(new TikaMimeTypes()));
- resourceProviders.add(new SingletonResourceProvider(new TikaDetectors()));
- resourceProviders.add(new SingletonResourceProvider(new TikaParsers()));
- resourceProviders.add(new SingletonResourceProvider(new TikaVersion()));
- if (tikaServerConfig.isEnableUnsecureFeatures()) {
- //check to make sure there are both fetchers and emitters
- //specified. It is possible that users may only specify fetchers
- //for legacy endpoints.
- if (tikaServerConfig
- .getSupportedFetchers()
- .size() > 0 && tikaServerConfig
- .getSupportedEmitters()
- .size() > 0) {
- addAsyncResource = true;
- addPipesResource = true;
- }
- resourceProviders.add(new SingletonResourceProvider(new TikaServerStatus(serverStatus)));
- }
- } else {
- for (String endPoint : tikaServerConfig.getEndpoints()) {
- if ("meta".equals(endPoint)) {
- resourceProviders.add(new SingletonResourceProvider(new MetadataResource()));
- } else if ("rmeta".equals(endPoint)) {
- resourceProviders.add(new SingletonResourceProvider(new RecursiveMetadataResource()));
- } else if ("detect".equals(endPoint)) {
- resourceProviders.add(new SingletonResourceProvider(new DetectorResource(serverStatus)));
- } else if ("language".equals(endPoint)) {
- resourceProviders.add(new SingletonResourceProvider(new LanguageResource()));
- } else if ("translate".equals(endPoint)) {
- resourceProviders.add(new SingletonResourceProvider(new TranslateResource(serverStatus, tikaServerConfig.getTaskTimeoutMillis())));
- } else if ("tika".equals(endPoint)) {
- resourceProviders.add(new SingletonResourceProvider(new TikaResource()));
- } else if ("unpack".equals(endPoint)) {
- resourceProviders.add(new SingletonResourceProvider(new UnpackerResource()));
- } else if ("mime".equals(endPoint)) {
- resourceProviders.add(new SingletonResourceProvider(new TikaMimeTypes()));
- } else if ("detectors".equals(endPoint)) {
- resourceProviders.add(new SingletonResourceProvider(new TikaDetectors()));
- } else if ("parsers".equals(endPoint)) {
- resourceProviders.add(new SingletonResourceProvider(new TikaParsers()));
- } else if ("version".equals(endPoint)) {
- resourceProviders.add(new SingletonResourceProvider(new TikaVersion()));
- } else if ("pipes".equals(endPoint)) {
- addPipesResource = true;
- } else if ("async".equals(endPoint)) {
- addAsyncResource = true;
- } else if ("status".equals(endPoint)) {
- resourceProviders.add(new SingletonResourceProvider(new TikaServerStatus(serverStatus)));
- }
- }
- }
-
- if (addAsyncResource) {
- final AsyncResource localAsyncResource = new AsyncResource(tikaServerConfig.getConfigPath(), tikaServerConfig.getSupportedFetchers());
- Runtime
- .getRuntime()
- .addShutdownHook(new Thread(() -> {
- try {
- localAsyncResource.shutdownNow();
- } catch (Exception e) {
- LOG.warn("problem shutting down local async resource", e);
- }
- }));
- resourceProviders.add(new SingletonResourceProvider(localAsyncResource));
- }
- if (addPipesResource) {
- final PipesResource localPipesResource = new PipesResource(tikaServerConfig.getConfigPath());
- Runtime
- .getRuntime()
- .addShutdownHook(new Thread(() -> {
- try {
- localPipesResource.close();
- } catch (Exception e) {
- LOG.warn("exception closing local pipes resource", e);
- }
- }));
- resourceProviders.add(new SingletonResourceProvider(localPipesResource));
- }
- resourceProviders.addAll(loadResourceServices(serverStatus));
- return resourceProviders;
- }
-
- private static void logFetchersAndEmitters(boolean enableUnsecureFeatures, FetcherManager fetcherManager, EmitterManager emitterManager) {
- if (enableUnsecureFeatures) {
- StringBuilder sb = new StringBuilder();
- Set<String> supportedFetchers = fetcherManager.getSupported();
- sb.append("enableSecureFeatures has been selected.\n");
- if (supportedFetchers.size() == 0) {
- sb.append("There are no fetchers specified in the TikaConfig");
- } else {
- sb.append("The following fetchers are available to whomever has " + "access to this server:\n");
- for (String p : supportedFetchers) {
- sb
- .append(p)
- .append("\n");
- }
- }
- Set<String> emitters = emitterManager.getSupported();
- if (supportedFetchers.size() == 0) {
- sb.append("There are no emitters specified in the TikaConfig");
- } else {
- sb.append("The following emitters are available to whomever has " + "access to this server:\n");
- for (String e : emitters) {
- sb
- .append(e)
- .append("\n");
- }
- }
- LOG.info(sb.toString());
- } else {
- if (emitterManager
- .getSupported()
- .size() > 0) {
- String warn = "enableUnsecureFeatures has not been set to 'true' in the server " + "config file.\n" + "The " + emitterManager
- .getSupported()
- .size() + " emitter(s) that you've\n" + "specified in TikaConfig will not be available on the /emit " + "or /async endpoints.\n" +
- "To enable your emitters, start tika-server with " + "<enableUnsecureFeatures>true</enableUnsecureFeatures> " + "parameter in " + "the TikaConfig\n\n";
- LOG.warn(warn);
- }
- if (emitterManager
- .getSupported()
- .size() > 0) {
- String warn = "enableUnsecureFeatures has not been set to 'true' in the server " + "config file.\n" + "The " + emitterManager
- .getSupported()
- .size() + " fetcher(s) that you've\n" + "specified in TikaConfig will not be available on the /emit " + "or /async endpoints.\n" +
- "To enable your emitters, start tika-server with " + "<enableUnsecureFeatures>true</enableUnsecureFeatures> " + "parameter in " + "the TikaConfig\n\n";
- LOG.warn(warn);
- }
- }
- }
-
- private static Collection<? extends ResourceProvider> loadResourceServices(ServerStatus serverStatus) {
- List<TikaServerResource> resources = new ServiceLoader(TikaServerProcess.class.getClassLoader()).loadServiceProviders(TikaServerResource.class);
- List<ResourceProvider> providers = new ArrayList<>();
- for (TikaServerResource r : resources) {
- LOG.info("loading resource from SPI: " + r.getClass());
- if (r instanceof ServerStatusResource) {
- ((ServerStatusResource) r).setServerStatus(serverStatus);
- }
- providers.add(new SingletonResourceProvider(r));
- }
- return providers;
- }
-
- private static Collection<?> loadWriterServices() {
- return new ServiceLoader(TikaServerProcess.class.getClassLoader()).loadServiceProviders(org.apache.tika.server.core.writer.TikaServerWriter.class);
- }
-
- private static class ServerDetails {
- JAXRSServerFactoryBean sf;
- String serverId;
- String url;
- ServerStatus serverStatus;
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java
deleted file mode 100644
index a2c973f66..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java
+++ /dev/null
@@ -1,459 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core;
-
-import static java.nio.file.StandardOpenOption.READ;
-import static java.nio.file.StandardOpenOption.WRITE;
-import static org.apache.tika.server.core.TikaServerCli.TIKA_SERVER_ID_ENV;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.PrintStream;
-import java.net.BindException;
-import java.nio.Buffer;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileLock;
-import java.nio.channels.OverlappingFileLockException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.Duration;
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.utils.ProcessUtils;
-
-public class TikaServerWatchDog implements Callable<WatchDogResult> {
-
- private static final Logger LOG = LoggerFactory.getLogger(TikaServerWatchDog.class);
- private static Set<Process> PROCESSES = ConcurrentHashMap.newKeySet();
- private static Set<ForkedProcess> FORKED_PROCESSES = ConcurrentHashMap.newKeySet();
-
- static {
- Thread shutdownHook = new Thread(() -> {
- //prioritize destroying processes
- for (Process process : PROCESSES) {
- process.destroyForcibly();
- }
- //once that's done, try to clean up tmp files too
- for (ForkedProcess forkedProcess : FORKED_PROCESSES) {
- try {
- forkedProcess.close();
- } catch (DoNotRestartException | InterruptedException e) {
- //swallow
- }
- }
- });
- Runtime
- .getRuntime()
- .addShutdownHook(shutdownHook);
- }
-
- private final int port;
- private final String id;
- private final TikaServerConfig tikaServerConfig;
- private final Object[] forkedStatusLock = new Object[0];
- private volatile FORKED_STATUS forkedStatus = FORKED_STATUS.INITIALIZING;
- private volatile Instant lastPing = null;
- private ForkedProcess forkedProcess = null;
- private int restarts = 0;
- private volatile boolean shutDown = false;
-
- TikaServerWatchDog(int port, String id, TikaServerConfig tikaServerConfig) {
- this.port = port;
- this.id = id;
- this.tikaServerConfig = tikaServerConfig;
- }
-
- private static void redirectIO(final InputStream src, final PrintStream targ) {
- Thread gobbler = new Thread(() -> {
- BufferedReader reader = new BufferedReader(new InputStreamReader(src, StandardCharsets.UTF_8));
- String line;
- try {
- line = reader.readLine();
- while (line != null) {
- targ.println(line);
- line = reader.readLine();
- }
- } catch (IOException e) {
- //swallow
- }
- });
- gobbler.setDaemon(true);
- gobbler.start();
- }
-
- private static synchronized void destroyForkedForcibly(Process process) throws InterruptedException {
-
- process = process.destroyForcibly();
- try {
- boolean destroyed = process.waitFor(60, TimeUnit.SECONDS);
-
- if (!destroyed) {
- LOG.error("Forked process still alive after 60 seconds. " + "Shutting down the forking process.");
- System.exit(1);
- }
- } finally {
- PROCESSES.remove(process);
- }
- }
-
- private static void closeForkedProcess(ForkedProcess forkedProcess) throws DoNotRestartException, InterruptedException {
- try {
- forkedProcess.close();
- } finally {
- FORKED_PROCESSES.remove(forkedProcess);
- }
- }
-
- @Override
- public WatchDogResult call() throws Exception {
- boolean mustRestart = true;
- try {
- while (true) {
- if (tikaServerConfig.getMaxRestarts() > 0 && restarts >= tikaServerConfig.getMaxRestarts()) {
- LOG.warn("hit max restarts ({}). Ending processing for {} {}", restarts, id, port);
- return new WatchDogResult(port, id, restarts);
- }
-
- try {
- if (mustRestart) {
- forkedProcess = startForkedProcess(restarts++);
- if (forkedProcess == null) {
- if (!shutDown) {
- throw new IllegalArgumentException("forked process should not be " + "null when not in shutdown mode");
- } else {
- return new WatchDogResult(port, id, restarts);
- }
- }
- setForkedStatus(FORKED_STATUS.RUNNING);
- mustRestart = false;
- }
- boolean exited = forkedProcess.process.waitFor(tikaServerConfig.getTaskPulseMillis(), TimeUnit.MILLISECONDS);
- if (exited) {
- LOG.info("forked process exited with exit value {}", forkedProcess.process.exitValue());
- closeForkedProcess(forkedProcess);
- mustRestart = true;
- } else {
- ForkedStatus status = forkedProcess.readStatus();
- if (status.status == FORKED_STATUS.FAILED_COMMUNICATION.ordinal()) {
- LOG.info("failed to read from status file. Restarting now.");
- closeForkedProcess(forkedProcess);
- mustRestart = true;
- } else if (status.status == FORKED_STATUS.SHUTTING_DOWN.ordinal()) {
- LOG.info("Forked process is in shutting down mode. Will wait a bit");
- forkedProcess.process.waitFor(tikaServerConfig.getTaskTimeoutMillis(), TimeUnit.MILLISECONDS);
- closeForkedProcess(forkedProcess);
- mustRestart = true;
- } else {
- long elapsed = Duration
- .between(Instant.ofEpochMilli(status.timestamp), Instant.now())
- .toMillis();
- if (elapsed > tikaServerConfig.getTaskTimeoutMillis()) {
- LOG.info("{} ms have elapsed since forked process " + "last updated status. " + "Shutting down and restarting.", elapsed);
- closeForkedProcess(forkedProcess);
- mustRestart = true;
- }
- }
- }
- } catch (InterruptedException e) {
- return new WatchDogResult(port, id, restarts);
- }
- }
- } finally {
- setForkedStatus(FORKED_STATUS.SHUTTING_DOWN);
- LOG.debug("about to shutdown");
- if (forkedProcess != null) {
- LOG.info("about to shutdown process");
- closeForkedProcess(forkedProcess);
- }
- }
- }
-
- public synchronized void close() throws DoNotRestartException, InterruptedException {
- setForkedStatus(FORKED_STATUS.SHUTTING_DOWN);
- LOG.debug("received 'close()'; about to shutdown");
- shutDown();
- closeForkedProcess(forkedProcess);
- }
-
- private synchronized ForkedProcess startForkedProcess(int restarts) throws Exception {
- LOG.debug("attempting to start forked process on {} restarts", restarts);
- int consecutiveRestarts = 0;
- //if there's a bind exception, retry for 30 seconds to give the OS
- //a chance to release the port
- int maxBind = 30;
- while (consecutiveRestarts < maxBind && !shutDown) {
- try {
- ForkedProcess forkedProcess = new ForkedProcess(restarts);
- FORKED_PROCESSES.add(forkedProcess);
- return forkedProcess;
- } catch (BindException e) {
- LOG.warn("WatchDog observes bind exception on retry {}. " + "Will retry {} times.", consecutiveRestarts, maxBind);
- consecutiveRestarts++;
- Thread.sleep(1000);
- if (consecutiveRestarts >= maxBind) {
- throw e;
- }
- }
- }
- if (shutDown) {
- return null;
- }
- throw new RuntimeException("Couldn't start forked process");
- }
-
-
- public void shutDown() {
- shutDown = true;
- }
-
- private void setForkedStatus(FORKED_STATUS status) {
- synchronized (forkedStatusLock) {
- forkedStatus = status;
- }
- }
-
- private enum FORKED_STATUS {
- INITIALIZING, RUNNING, SHUTTING_DOWN, FAILED_COMMUNICATION
- }
-
- private static class ForkedStatus {
- private final long timestamp;
- private final int status;
- private final int numTasks;
-
- public ForkedStatus(long timestamp, int status, int numTasks) {
- this.timestamp = timestamp;
- this.status = status;
- this.numTasks = numTasks;
- }
-
- @Override
- public String toString() {
- return "ForkedStatus{" + "timestamp=" + timestamp + ", status=" + status + ", numTasks=" + numTasks + '}';
- }
- }
-
- private static class DoNotRestartException extends TikaException {
-
- public DoNotRestartException(String msg) {
- super(msg);
- }
-
- public DoNotRestartException(String msg, Throwable cause) {
- super(msg, cause);
- }
- }
-
- private class ForkedProcess {
- private final Process process;
- // private final DataOutputStream toForked;
- private final Path forkedStatusFile;
- private final ByteBuffer statusBuffer = ByteBuffer.allocate(16);
-
- private ForkedProcess(int numRestarts) throws Exception {
- String prefix = tikaServerConfig.getTempFilePrefix();
-
- this.forkedStatusFile = Files.createTempFile(prefix, "");
- this.process = startProcess(numRestarts, forkedStatusFile);
-
- //wait for file to be written/initialized by forked process
- Instant start = Instant.now();
- long elapsed = Duration
- .between(start, Instant.now())
- .toMillis();
- try {
- while (process.isAlive() && Files.size(forkedStatusFile) < 12 && elapsed < tikaServerConfig.getMaxForkedStartupMillis()) {
- Thread.sleep(50);
- elapsed = Duration
- .between(start, Instant.now())
- .toMillis();
- }
- } catch (IOException e) {
- //the forkedStatusFile can be deleted by the
- //forked process if it closes...this can lead to a NoSuchFileException
- LOG.warn("failed to start forked process", e);
- }
-
- if (elapsed > tikaServerConfig.getMaxForkedStartupMillis()) {
- close();
- throw new RuntimeException("Forked process failed to start after " + elapsed + " (ms)");
- }
- if (!process.isAlive()) {
- close();
- if (process.exitValue() == TikaServerProcess.BIND_EXCEPTION) {
- throw new BindException("couldn't bind");
- }
- throw new RuntimeException("Failed to start forked process -- forked is not alive");
- }
- if (!Files.exists(forkedStatusFile)) {
- close();
- throw new RuntimeException("Failed to start forked process -- forked status file does not exist");
- }
-
- lastPing = Instant.now();
- }
-
- private ForkedStatus readStatus() throws Exception {
- Instant started = Instant.now();
- long elapsed = Duration
- .between(started, Instant.now())
- .toMillis();
- //only reading, but need to include write to allow for locking
- try (FileChannel fc = FileChannel.open(forkedStatusFile, READ, WRITE)) {
-
- while (elapsed < tikaServerConfig.getTaskTimeoutMillis()) {
- try (FileLock lock = fc.tryLock(0, 16, true)) {
- if (lock != null) {
- ((Buffer) statusBuffer).position(0);
- fc.read(statusBuffer);
- long timestamp = statusBuffer.getLong(0);
- int status = statusBuffer.getInt(8);
- int numTasks = statusBuffer.getInt(12);
- return new ForkedStatus(timestamp, status, numTasks);
- }
- } catch (OverlappingFileLockException e) {
- //swallow
- }
- Thread.sleep(100);
- elapsed = Duration
- .between(started, Instant.now())
- .toMillis();
- }
- }
- return new ForkedStatus(-1, FORKED_STATUS.FAILED_COMMUNICATION.ordinal(), -1);
- }
-
- private void close() throws DoNotRestartException, InterruptedException {
-
- try {
- if (!process.isAlive()) {
- try {
- int exit = process.exitValue();
- if (exit == TikaServerProcess.DO_NOT_RESTART_EXIT_VALUE) {
- throw new DoNotRestartException("Forked exited with: " + exit);
- }
- } catch (IllegalThreadStateException e) {
- //swallow
- }
- }
- destroyForkedForcibly(process);
- } finally {
- if (forkedStatusFile != null) {
- try {
- if (Files.isRegularFile(forkedStatusFile)) {
- Files.delete(forkedStatusFile);
- }
- LOG.debug("deleted " + forkedStatusFile);
- } catch (IOException e) {
- LOG.warn("problem deleting forked process status file", e);
- }
- }
- }
- }
-
- private Process startProcess(int numRestarts, Path forkedStatusFile) throws IOException {
- ProcessBuilder builder = new ProcessBuilder();
- builder.redirectError(ProcessBuilder.Redirect.INHERIT);
-
- List<String> argList = new ArrayList<>();
- String javaPath = tikaServerConfig.getJavaPath();
- List<String> jvmArgs = tikaServerConfig.getForkedJvmArgs();
- List<String> forkedArgs = tikaServerConfig.getForkedProcessArgs(port, id);
- forkedArgs.add("-forkedStatusFile");
- forkedArgs.add(ProcessUtils.escapeCommandLine(forkedStatusFile
- .toAbsolutePath()
- .toString()));
-
- argList.add(javaPath);
- if (!jvmArgs.contains("-cp") && !jvmArgs.contains("--classpath")) {
- String cp = System.getProperty("java.class.path");
- jvmArgs.add("-cp");
- jvmArgs.add(cp);
- }
- //this is mostly for log4j 1.x so that different processes
- //can log to different log files
- jvmArgs.add("-Dtika.server.id=" + tikaServerConfig.getId());
- argList.addAll(jvmArgs);
-
- argList.add("org.apache.tika.server.core.TikaServerProcess");
- argList.addAll(forkedArgs);
-
- argList.add("-numRestarts");
- argList.add(Integer.toString(numRestarts));
- LOG.debug("forked process commandline: " + argList.toString());
- builder.command(argList);
- //now overwrite with the specific server id
- //this is mostly for log4j 2.x so that different processes
- //can log to different log files via {env:tika.server.id}
- builder
- .environment()
- .put(TIKA_SERVER_ID_ENV, id);
- Process process = builder.start();
- PROCESSES.add(process);
- //redirect stdout to parent stderr to avoid error msgs
- //from maven during build: Corrupted STDOUT by directly writing to
- // native stream in forked
- redirectIO(process.getInputStream(), System.err);
- redirectIO(process.getErrorStream(), System.err);
- return process;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- ForkedProcess that = (ForkedProcess) o;
-
- if (!process.equals(that.process)) {
- return false;
- }
- if (!forkedStatusFile.equals(that.forkedStatusFile)) {
- return false;
- }
- return statusBuffer.equals(that.statusBuffer);
- }
-
- @Override
- public int hashCode() {
- int result = process.hashCode();
- result = 31 * result + forkedStatusFile.hashCode();
- result = 31 * result + statusBuffer.hashCode();
- return result;
- }
- }
-
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TlsConfig.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TlsConfig.java
deleted file mode 100644
index df0ddfc70..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/TlsConfig.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core;
-
-import java.util.Map;
-
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.utils.StringUtils;
-
-public class TlsConfig implements Initializable {
-
- //TODO make this configurable
- private final boolean passwordsAESEncrypted = false;
- private boolean active = false;
- private String keyStoreType = null;
- private String keyStorePassword = null;
- private String keyStoreFile = null;
- private String trustStoreType = null;
- private String trustStorePassword = null;
- private String trustStoreFile = null;
-
- private boolean clientAuthenticationWanted = false;
-
- private boolean clientAuthenticationRequired = false;
-
- public boolean isActive() {
- return active;
- }
-
- public void setActive(boolean active) {
- this.active = active;
- }
-
- public boolean isPasswordsAESEncrypted() {
- return passwordsAESEncrypted;
- }
-
- public String getKeyStoreType() {
- return keyStoreType;
- }
-
- public void setKeyStoreType(String keyStoreType) {
- this.keyStoreType = keyStoreType;
- }
-
- public String getKeyStorePassword() {
- return keyStorePassword;
- }
-
- public void setKeyStorePassword(String keyStorePassword) {
- this.keyStorePassword = keyStorePassword;
- }
-
- public String getKeyStoreFile() {
- return keyStoreFile;
- }
-
- public void setKeyStoreFile(String keyStoreFile) {
- this.keyStoreFile = keyStoreFile;
- }
-
- public String getTrustStoreType() {
- return trustStoreType;
- }
-
- public void setTrustStoreType(String trustStoreType) {
- this.trustStoreType = trustStoreType;
- }
-
- public String getTrustStorePassword() {
- return trustStorePassword;
- }
-
- public void setTrustStorePassword(String trustStorePassword) {
- this.trustStorePassword = trustStorePassword;
- }
-
- public String getTrustStoreFile() {
- return trustStoreFile;
- }
-
- public void setTrustStoreFile(String trustStoreFile) {
- this.trustStoreFile = trustStoreFile;
- }
-
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
-
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
- if (active) {
- if (StringUtils.isBlank(keyStoreType)) {
- throw new TikaConfigException("must initialize keyStoreType");
- } else if (StringUtils.isBlank(keyStoreFile)) {
- throw new TikaConfigException("must initialize keyStoreFile");
- } else if (StringUtils.isBlank(keyStorePassword)) {
- throw new TikaConfigException("must initialize keyStorePassword");
- }
- if (hasTrustStore()) {
- if (StringUtils.isBlank(trustStoreType)) {
- throw new TikaConfigException("must initialize trustStoreType " + "if there's any trustStore info");
- } else if (StringUtils.isBlank(trustStoreFile)) {
- throw new TikaConfigException("must initialize trustStoreFile " + "if there's any trustStore info");
- } else if (StringUtils.isBlank(trustStorePassword)) {
- throw new TikaConfigException("must initialize trustStorePassword " + "if there's any trustStore info");
- }
- }
- if (!hasTrustStore() && isClientAuthenticationRequired()) {
- throw new TikaConfigException("requiring client authentication, but no trust " + "store has been specified?!");
- }
- }
- }
-
- public boolean isClientAuthenticationWanted() {
- return clientAuthenticationWanted;
- }
-
- public void setClientAuthenticationWanted(boolean clientAuthenticationWanted) {
- this.clientAuthenticationWanted = clientAuthenticationWanted;
- }
-
- public boolean isClientAuthenticationRequired() {
- return clientAuthenticationRequired;
- }
-
- public void setClientAuthenticationRequired(boolean clientAuthenticationRequired) {
- this.clientAuthenticationRequired = clientAuthenticationRequired;
- }
-
- @Override
- public String toString() {
- return "TlsConfig{" + "active=" + active + ", passwordsAESEncrypted=" + passwordsAESEncrypted + ", keyStoreType='" + keyStoreType + '\'' + ", keyStorePassword='" +
- keyStorePassword + '\'' + ", keyStoreFile='" + keyStoreFile + '\'' + ", trustStoreType='" + trustStoreType + '\'' + ", trustStorePassword='" + trustStorePassword +
- '\'' + ", trustStoreFile='" + trustStoreFile + '\'' + ", clientAuthenticationWanted=" + clientAuthenticationWanted + ", isClientAuthenticationRequired=" +
- clientAuthenticationRequired + '}';
- }
-
- public boolean hasTrustStore() {
- return !StringUtils.isBlank(trustStoreType) && !StringUtils.isBlank(trustStorePassword) && !StringUtils.isBlank(trustStoreFile);
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/WatchDogResult.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/WatchDogResult.java
deleted file mode 100644
index 4471ff605..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/WatchDogResult.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core;
-
-public class WatchDogResult {
-
- private final int port;
- private final String id;
- private final int numRetries;
-
- public WatchDogResult(int port, String id, int numRetries) {
- this.port = port;
- this.id = id;
- this.numRetries = numRetries;
- }
-
- public int getPort() {
- return port;
- }
-
- public int getNumRestarts() {
- return numRetries;
- }
-
- public String getId() {
- return id;
- }
-
- @Override
- public String toString() {
- return "WatchDogResult{" + "port=" + port + ", id='" + id + '\'' + ", numRetries=" + numRetries + '}';
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/config/DocumentSelectorConfig.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/config/DocumentSelectorConfig.java
deleted file mode 100644
index 27a2c868b..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/config/DocumentSelectorConfig.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core.config;
-
-import jakarta.ws.rs.core.MultivaluedMap;
-import org.apache.commons.lang3.Strings;
-
-import org.apache.tika.extractor.DocumentSelector;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.server.core.ParseContextConfig;
-
-public class DocumentSelectorConfig implements ParseContextConfig {
-
- public static final String X_TIKA_SKIP_EMBEDDED_HEADER = "X-Tika-Skip-Embedded";
-
- @Override
- public void configure(MultivaluedMap<String, String> httpHeaders, Metadata mtadata, ParseContext context) {
- DocumentSelector documentSelector = null;
- for (String key : httpHeaders.keySet()) {
- if (Strings.CS.endsWith(key, X_TIKA_SKIP_EMBEDDED_HEADER)) {
- String skipEmbedded = httpHeaders.getFirst(key);
- if (Boolean.parseBoolean(skipEmbedded)) {
- documentSelector = metadata -> false;
- }
- }
- }
- if (documentSelector != null) {
- context.set(DocumentSelector.class, documentSelector);
- }
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/config/PasswordProviderConfig.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/config/PasswordProviderConfig.java
deleted file mode 100644
index 60438f19e..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/config/PasswordProviderConfig.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core.config;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import jakarta.ws.rs.core.MultivaluedMap;
-import org.apache.commons.codec.binary.Base64;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.PasswordProvider;
-import org.apache.tika.server.core.ParseContextConfig;
-
-public class PasswordProviderConfig implements ParseContextConfig {
- public static final String PASSWORD = "Password";
- public static final String PASSWORD_BASE64_UTF8 = "Password_Base64_UTF-8";
- private static final Base64 BASE_64 = new Base64();
-
- private static String decodeBase64UTF8(String s) {
- byte[] bytes = BASE_64.decode(s);
- return new String(bytes, UTF_8);
- }
-
- @Override
- public void configure(MultivaluedMap<String, String> httpHeaders, Metadata metadata, ParseContext context) {
- String tmpPassword = httpHeaders.getFirst(PASSWORD_BASE64_UTF8);
- if (tmpPassword != null) {
- tmpPassword = decodeBase64UTF8(tmpPassword);
- } else {
- tmpPassword = httpHeaders.getFirst(PASSWORD);
- }
- if (tmpPassword != null) {
- final String password = tmpPassword;
- context.set(PasswordProvider.class, metadata1 -> password);
- }
- }
-
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/config/TimeoutConfig.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/config/TimeoutConfig.java
deleted file mode 100644
index 88a9e102f..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/config/TimeoutConfig.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core.config;
-
-import jakarta.ws.rs.core.MultivaluedMap;
-
-import org.apache.tika.config.TikaTaskTimeout;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.server.core.ParseContextConfig;
-
-public class TimeoutConfig implements ParseContextConfig {
-
- public static final String X_TIKA_TIMEOUT_MILLIS = "X-Tika-Timeout-Millis";
-
- @Override
- public void configure(MultivaluedMap<String, String> httpHeaders, Metadata metadata, ParseContext context) {
- if (httpHeaders.containsKey(X_TIKA_TIMEOUT_MILLIS)) {
- long timeout = Long.parseLong(httpHeaders.getFirst(X_TIKA_TIMEOUT_MILLIS));
- context.set(TikaTaskTimeout.class, new TikaTaskTimeout(timeout));
- }
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncRequest.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncRequest.java
deleted file mode 100644
index 425b97f63..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncRequest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core.resource;
-
-import java.util.List;
-
-import org.apache.tika.pipes.core.FetchEmitTuple;
-
-public class AsyncRequest {
- private final List<FetchEmitTuple> tuples;
-
- public AsyncRequest(List<FetchEmitTuple> tuples) {
- this.tuples = tuples;
- }
-
- public List<FetchEmitTuple> getTuples() {
- return tuples;
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java
deleted file mode 100644
index aa2160fc5..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core.resource;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ArrayBlockingQueue;
-
-import jakarta.ws.rs.BadRequestException;
-import jakarta.ws.rs.POST;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.Produces;
-import jakarta.ws.rs.core.Context;
-import jakarta.ws.rs.core.HttpHeaders;
-import jakarta.ws.rs.core.UriInfo;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.async.AsyncProcessor;
-import org.apache.tika.pipes.core.async.OfferLargerThanQueueSize;
-import org.apache.tika.pipes.core.emitter.EmitData;
-import org.apache.tika.pipes.core.emitter.EmitterManager;
-import org.apache.tika.pipes.core.extractor.EmbeddedDocumentBytesConfig;
-import org.apache.tika.pipes.core.fetcher.FetchKey;
-import org.apache.tika.pipes.core.serialization.JsonFetchEmitTupleList;
-
-@Path("/async")
-public class AsyncResource {
-
- private static final Logger LOG = LoggerFactory.getLogger(AsyncResource.class);
- private final AsyncProcessor asyncProcessor;
- private final Set<String> supportedFetchers;
- private final EmitterManager emitterManager;
- long maxQueuePauseMs = 60000;
- private ArrayBlockingQueue<FetchEmitTuple> queue;
-
- public AsyncResource(java.nio.file.Path tikaConfigPath, Set<String> supportedFetchers) throws TikaException, IOException, SAXException {
- this.asyncProcessor = new AsyncProcessor(tikaConfigPath);
- this.supportedFetchers = supportedFetchers;
- this.emitterManager = EmitterManager.load(tikaConfigPath);
- }
-
- public ArrayBlockingQueue<FetchEmitTuple> getFetchEmitQueue(int queueSize) {
- this.queue = new ArrayBlockingQueue<>(queueSize);
- return queue;
- }
-
- public ArrayBlockingQueue<EmitData> getEmitDataQueue(int size) {
- return new ArrayBlockingQueue<>(size);
- }
-
- /**
- * The client posts a json request. At a minimum, this must be a
- * json object that contains an emitter and a fetcherString key with
- * the key to fetch the inputStream. Optionally, it may contain a metadata
- * object that will be used to populate the metadata key for pass
- * through of metadata from the client.
- * <p>
- * The extracted text content is stored with the key
- * {@link TikaCoreProperties#TIKA_CONTENT}
- * <p>
- * Must specify a fetcherString and an emitter in the posted json.
- *
- * @param info uri info
- * @return InputStream that can be deserialized as a list of {@link Metadata} objects
- * @throws Exception
- */
- @POST
- @Produces("application/json")
- public Map<String, Object> post(InputStream is, @Context HttpHeaders httpHeaders, @Context UriInfo info) throws Exception {
-
- AsyncRequest request = deserializeASyncRequest(is);
-
- //make sure that there are no problems with
- //the requested fetchers and emitters
- //throw early
- for (FetchEmitTuple t : request.getTuples()) {
- if (!supportedFetchers.contains(t
- .getFetchKey()
- .getFetcherName())) {
- return badFetcher(t.getFetchKey());
- }
- if (!emitterManager
- .getSupported()
- .contains(t
- .getEmitKey()
- .getEmitterName())) {
- return badEmitter(t
- .getEmitKey()
- .getEmitterName());
- }
- ParseContext parseContext = t.getParseContext();
- EmbeddedDocumentBytesConfig embeddedDocumentBytesConfig = parseContext.get(EmbeddedDocumentBytesConfig.class);
- if (embeddedDocumentBytesConfig != null && embeddedDocumentBytesConfig.isExtractEmbeddedDocumentBytes() &&
- !StringUtils.isAllBlank(embeddedDocumentBytesConfig.getEmitter())) {
- String bytesEmitter = embeddedDocumentBytesConfig.getEmitter();
- if (!emitterManager
- .getSupported()
- .contains(bytesEmitter)) {
- return badEmitter(bytesEmitter);
- }
- }
- }
- //Instant start = Instant.now();
- try {
- boolean offered = asyncProcessor.offer(request.getTuples(), maxQueuePauseMs);
- if (offered) {
- LOG.info("accepted {} tuples, capacity={}", request
- .getTuples()
- .size(), asyncProcessor.getCapacity());
- return ok(request
- .getTuples()
- .size());
- } else {
- LOG.info("throttling {} tuples, capacity={}", request
- .getTuples()
- .size(), asyncProcessor.getCapacity());
- return throttle(request
- .getTuples()
- .size());
- }
- } catch (OfferLargerThanQueueSize e) {
- LOG.info("throttling {} tuples, capacity={}", request
- .getTuples()
- .size(), asyncProcessor.getCapacity());
- return throttle(request
- .getTuples()
- .size());
- }
- }
-
- private Map<String, Object> ok(int size) {
- Map<String, Object> map = new HashMap<>();
- map.put("status", "ok");
- map.put("added", size);
- return map;
- }
-
- private Map<String, Object> throttle(int requestSize) {
- Map<String, Object> map = new HashMap<>();
- map.put("status", "throttled");
- map.put("msg", "not able to receive request of size " + requestSize + " at this time");
- map.put("capacity", asyncProcessor.getCapacity());
- return map;
- }
-
- private Map<String, Object> badEmitter(String emitterName) {
- throw new BadRequestException("can't find emitter for " + emitterName);
- }
-
- private Map<String, Object> badFetcher(FetchKey fetchKey) {
- throw new BadRequestException("can't find fetcher for " + fetchKey.getFetcherName());
- }
-
- private AsyncRequest deserializeASyncRequest(InputStream is) throws IOException {
- try (Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8)) {
- return new AsyncRequest(JsonFetchEmitTupleList.fromJson(reader));
- }
- }
-
- public void shutdownNow() throws Exception {
- asyncProcessor.close();
- }
-
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/LanguageResource.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/LanguageResource.java
deleted file mode 100644
index 8e2a2b1df..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/LanguageResource.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core.resource;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import jakarta.ws.rs.Consumes;
-import jakarta.ws.rs.POST;
-import jakarta.ws.rs.PUT;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.Produces;
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.langdetect.optimaize.OptimaizeLangDetector;
-import org.apache.tika.language.detect.LanguageResult;
-
-@Path("/language")
-public class LanguageResource {
- private static final Logger LOG = LoggerFactory.getLogger(LanguageResource.class);
-
- @PUT
- @POST
- @Path("/stream")
- @Consumes("*/*")
- @Produces("text/plain")
- public String detect(final InputStream is) throws IOException {
- String fileTxt = IOUtils.toString(is, UTF_8);
- LanguageResult language = new OptimaizeLangDetector()
- .loadModels()
- .detect(fileTxt);
- String detectedLang = language.getLanguage();
- LOG.info("Detecting language for incoming resource: [{}]", detectedLang);
- return detectedLang;
- }
-
- @PUT
- @POST
- @Path("/string")
- @Consumes("*/*")
- @Produces("text/plain")
- public String detect(final String string) throws IOException {
- LanguageResult language = new OptimaizeLangDetector()
- .loadModels()
- .detect(string);
- String detectedLang = language.getLanguage();
- LOG.info("Detecting language for incoming resource: [{}]", detectedLang);
- return detectedLang;
- }
-
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/MetadataResource.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/MetadataResource.java
deleted file mode 100644
index 29e452d94..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/MetadataResource.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core.resource;
-
-import static org.apache.tika.server.core.resource.TikaResource.fillMetadata;
-import static org.apache.tika.server.core.resource.TikaResource.fillParseContext;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import jakarta.ws.rs.Consumes;
-import jakarta.ws.rs.POST;
-import jakarta.ws.rs.PUT;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.PathParam;
-import jakarta.ws.rs.Produces;
-import jakarta.ws.rs.core.Context;
-import jakarta.ws.rs.core.HttpHeaders;
-import jakarta.ws.rs.core.MultivaluedMap;
-import jakarta.ws.rs.core.Response;
-import jakarta.ws.rs.core.UriInfo;
-import org.apache.cxf.jaxrs.ext.multipart.Attachment;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.extractor.DocumentSelector;
-import org.apache.tika.language.detect.LanguageHandler;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-
-
-@Path("/meta")
-public class MetadataResource {
- private static final Logger LOG = LoggerFactory.getLogger(MetadataResource.class);
-
- @POST
- @Consumes("multipart/form-data")
- @Produces({"text/csv", "application/json"})
- @Path("form")
- public Response getMetadataFromMultipart(Attachment att, @Context UriInfo info) throws Exception {
- return Response
- .ok(parseMetadata(att.getObject(InputStream.class), new Metadata(), att.getHeaders(), info))
- .build();
- }
-
- @PUT
- @Produces({"text/csv", "application/json"})
- public Response getMetadata(InputStream is, @Context HttpHeaders httpHeaders, @Context UriInfo info) throws Exception {
- Metadata metadata = new Metadata();
- return Response
- .ok(parseMetadata(TikaResource.getInputStream(is, metadata, httpHeaders, info), metadata, httpHeaders.getRequestHeaders(), info))
- .build();
- }
-
- /**
- * Get a specific metadata field. If the input stream cannot be parsed, but a
- * value was found for the given metadata field, then the value of the field
- * is returned as part of a 200 OK response; otherwise a
- * {@link javax.ws.rs.core.Response.Status#BAD_REQUEST} is generated. If the stream
- * was successfully parsed but the specific metadata field was not found, then a
- * {@link javax.ws.rs.core.Response.Status#NOT_FOUND} is returned.
- * <p/>
- * Note that this method handles multivalue fields and returns possibly more
- * metadata value than requested.
- * <p/>
- * If you want XMP, you must be careful to specify the exact XMP key.
- * For example, "Author" will return nothing, but "dc:creator" will return the correct value.
- *
- * @param is inputstream
- * @param httpHeaders httpheaders
- * @param info info
- * @param field the tika metadata field name
- * @return one of {@link javax.ws.rs.core.Response.Status#OK},
- * {@link javax.ws.rs.core.Response.Status#NOT_FOUND}, or
- * {@link javax.ws.rs.core.Response.Status#BAD_REQUEST}
- * @throws Exception
- */
- @PUT
- @Path("{field}")
- @Produces({"text/csv", "application/json", "text/plain"})
- public Response getMetadataField(InputStream is, @Context HttpHeaders httpHeaders, @Context UriInfo info, @PathParam("field") String field) throws Exception {
-
- // use BAD request to indicate that we may not have had enough data to
- // process the request
- Response.Status defaultErrorResponse = Response.Status.BAD_REQUEST;
- Metadata metadata = new Metadata();
- boolean success = false;
- try {
- parseMetadata(TikaResource.getInputStream(is, metadata, httpHeaders, info), metadata, httpHeaders.getRequestHeaders(), info);
- // once we've parsed the document successfully, we should use NOT_FOUND
- // if we did not see the field
- defaultErrorResponse = Response.Status.NOT_FOUND;
- success = true;
- } catch (Exception e) {
- LOG.info("Failed to process field {}", field, e);
- }
-
- if (success == false || metadata.get(field) == null) {
- return Response
- .status(defaultErrorResponse)
- .entity("Failed to get metadata field " + field)
- .build();
- }
-
- // remove fields we don't care about for the response
- for (String name : metadata.names()) {
- if (!field.equals(name)) {
- metadata.remove(name);
- }
- }
- return Response
- .ok(metadata)
- .build();
- }
-
- protected Metadata parseMetadata(InputStream is, Metadata metadata, MultivaluedMap<String, String> httpHeaders, UriInfo info) throws IOException {
- final ParseContext context = new ParseContext();
- Parser parser = TikaResource.createParser();
- fillMetadata(parser, metadata, httpHeaders);
- fillParseContext(httpHeaders, metadata, context);
- //no need to parse embedded docs
- context.set(DocumentSelector.class, metadata1 -> false);
-
- TikaResource.logRequest(LOG, "/meta", metadata);
- TikaResource.parse(parser, LOG, info.getPath(), is, new LanguageHandler() {
- public void endDocument() {
- metadata.set("language", getLanguage().getLanguage());
- }
- }, metadata, context);
- return metadata;
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesResource.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesResource.java
deleted file mode 100644
index fd12bc81c..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesResource.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core.resource;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.Map;
-
-import jakarta.ws.rs.POST;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.Produces;
-import jakarta.ws.rs.core.Context;
-import jakarta.ws.rs.core.HttpHeaders;
-import jakarta.ws.rs.core.UriInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.PipesConfig;
-import org.apache.tika.pipes.core.PipesException;
-import org.apache.tika.pipes.core.PipesParser;
-import org.apache.tika.pipes.core.PipesResult;
-import org.apache.tika.pipes.core.serialization.JsonFetchEmitTuple;
-
-@Path("/pipes")
-public class PipesResource {
-
-
- private static final Logger LOG = LoggerFactory.getLogger(PipesResource.class);
-
- private final PipesParser pipesParser;
-
- public PipesResource(java.nio.file.Path tikaConfig) throws TikaConfigException, IOException {
- PipesConfig pipesConfig = PipesConfig.load(tikaConfig);
- //this has to be zero. everything must be emitted through the PipesServer
- long maxEmit = pipesConfig.getMaxForEmitBatchBytes();
- if (maxEmit != 0) {
- pipesConfig.setMaxForEmitBatchBytes(0);
- if (maxEmit != PipesConfig.DEFAULT_MAX_FOR_EMIT_BATCH) {
- LOG.warn("resetting max for emit batch to 0");
- }
- }
- this.pipesParser = new PipesParser(pipesConfig);
- }
-
-
- /**
- * The client posts a json request. At a minimum, this must be a
- * json object that contains an emitter and a fetcherString key with
- * the key to fetch the inputStream. Optionally, it may contain a metadata
- * object that will be used to populate the metadata key for pass
- * through of metadata from the client. It may also include a handler config.
- * <p>
- * The extracted text content is stored with the key
- * {@link TikaCoreProperties#TIKA_CONTENT}
- * <p>
- * Must specify a fetcherString and an emitter in the posted json.
- *
- * @param info uri info
- * @return InputStream that can be deserialized as a list of {@link Metadata} objects
- * @throws Exception
- */
- @POST
- @Produces("application/json")
- public Map<String, String> postRmeta(InputStream is, @Context HttpHeaders httpHeaders, @Context UriInfo info) throws Exception {
- FetchEmitTuple t = null;
- try (Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8)) {
- t = JsonFetchEmitTuple.fromJson(reader);
- }
- return processTuple(t);
- }
-
- private Map<String, String> processTuple(FetchEmitTuple fetchEmitTuple) throws InterruptedException, PipesException, IOException {
-
- PipesResult pipesResult = pipesParser.parse(fetchEmitTuple);
- switch (pipesResult.getStatus()) {
- case CLIENT_UNAVAILABLE_WITHIN_MS:
- throw new IllegalStateException("client not available within " + "allotted amount of time");
- case EMIT_EXCEPTION:
- return returnEmitException(pipesResult.getMessage());
- case PARSE_SUCCESS:
- case PARSE_SUCCESS_WITH_EXCEPTION:
- throw new IllegalArgumentException("Should have emitted in forked process?!");
- case EMIT_SUCCESS:
- return returnSuccess();
- case EMIT_SUCCESS_PARSE_EXCEPTION:
- return parseException(pipesResult.getMessage(), true);
- case PARSE_EXCEPTION_EMIT:
- throw new IllegalArgumentException("Should have tried to emit in forked " + "process?!");
- case PARSE_EXCEPTION_NO_EMIT:
- return parseException(pipesResult.getMessage(), false);
- case TIMEOUT:
- return returnError("timeout");
- case OOM:
- return returnError("oom");
- case UNSPECIFIED_CRASH:
- return returnError("unknown_crash");
- case NO_EMITTER_FOUND: {
- throw new IllegalArgumentException("Couldn't find emitter that matched: " + fetchEmitTuple
- .getEmitKey()
- .getEmitterName());
- }
- default:
- throw new IllegalArgumentException("I'm sorry, I don't yet handle a status of " + "this type: " + pipesResult.getStatus());
- }
- }
-
- private Map<String, String> parseException(String msg, boolean emitted) {
- Map<String, String> statusMap = new HashMap<>();
- statusMap.put("status", "ok");
- statusMap.put("parse_exception", msg);
- statusMap.put("emitted", Boolean.toString(emitted));
- return statusMap;
- }
-
- private Map<String, String> returnEmitException(String msg) {
- Map<String, String> statusMap = new HashMap<>();
- statusMap.put("status", "emit_exception");
- statusMap.put("message", msg);
- return statusMap;
- }
-
- private Map<String, String> returnSuccess() {
- Map<String, String> statusMap = new HashMap<>();
- statusMap.put("status", "ok");
- return statusMap;
- }
-
- private Map<String, String> returnError(String type) {
- Map<String, String> statusMap = new HashMap<>();
- statusMap.put("status", "parse_error");
- statusMap.put("parse_error", type);
- return statusMap;
- }
-
- public void close() throws IOException {
- pipesParser.close();
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
deleted file mode 100644
index 298620076..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core.resource;
-
-import static org.apache.tika.server.core.resource.TikaResource.fillMetadata;
-import static org.apache.tika.server.core.resource.TikaResource.fillParseContext;
-import static org.apache.tika.server.core.resource.TikaResource.getConfig;
-
-import java.io.InputStream;
-import java.util.List;
-
-import jakarta.ws.rs.Consumes;
-import jakarta.ws.rs.POST;
-import jakarta.ws.rs.PUT;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.PathParam;
-import jakarta.ws.rs.Produces;
-import jakarta.ws.rs.WebApplicationException;
-import jakarta.ws.rs.core.Context;
-import jakarta.ws.rs.core.HttpHeaders;
-import jakarta.ws.rs.core.MultivaluedMap;
-import jakarta.ws.rs.core.Response;
-import jakarta.ws.rs.core.UriInfo;
-import org.apache.cxf.jaxrs.ext.multipart.Attachment;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.listfilter.MetadataListFilter;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.apache.tika.parser.RecursiveParserWrapper;
-import org.apache.tika.pipes.core.HandlerConfig;
-import org.apache.tika.sax.BasicContentHandlerFactory;
-import org.apache.tika.sax.RecursiveParserWrapperHandler;
-import org.apache.tika.server.core.MetadataList;
-import org.apache.tika.server.core.TikaServerParseException;
-
-@Path("/rmeta")
-public class RecursiveMetadataResource {
-
- protected static final String HANDLER_TYPE_PARAM = "handler";
- protected static final BasicContentHandlerFactory.HANDLER_TYPE DEFAULT_HANDLER_TYPE = BasicContentHandlerFactory.HANDLER_TYPE.XML;
- private static final Logger LOG = LoggerFactory.getLogger(RecursiveMetadataResource.class);
-
- public static List<Metadata> parseMetadata(InputStream is, Metadata metadata, MultivaluedMap<String, String> httpHeaders, UriInfo info, HandlerConfig handlerConfig)
- throws Exception {
-
- final ParseContext context = new ParseContext();
- Parser parser = TikaResource.createParser();
-
- RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
- fillMetadata(parser, metadata, httpHeaders);
- fillParseContext(httpHeaders, metadata, context);
- TikaResource.logRequest(LOG, "/rmeta", metadata);
-
- BasicContentHandlerFactory.HANDLER_TYPE type = handlerConfig.getType();
- RecursiveParserWrapperHandler handler =
- new RecursiveParserWrapperHandler(new BasicContentHandlerFactory(type, handlerConfig.getWriteLimit(), handlerConfig.isThrowOnWriteLimitReached(), context),
- handlerConfig.getMaxEmbeddedResources(), TikaResource
- .getConfig()
- .getMetadataFilter());
- try {
- TikaResource.parse(wrapper, LOG, "/rmeta", is, handler, metadata, context);
- } catch (TikaServerParseException e) {
- //do nothing
- LOG.debug("server parse exception", e);
- } catch (SecurityException | WebApplicationException e) {
- throw e;
- } catch (Exception e) {
- //we shouldn't get here?
- LOG.error("something went seriously wrong", e);
- }
- MetadataListFilter metadataListFilter = context.get(MetadataListFilter.class, getConfig().getMetadataListFilter());
- return metadataListFilter.filter(handler.getMetadataList());
- }
-
- static HandlerConfig buildHandlerConfig(MultivaluedMap<String, String> httpHeaders, String handlerTypeName, HandlerConfig.PARSE_MODE parseMode) {
- int writeLimit = -1;
- if (httpHeaders.containsKey("writeLimit")) {
- writeLimit = Integer.parseInt(httpHeaders.getFirst("writeLimit"));
- }
-
- int maxEmbeddedResources = -1;
- if (httpHeaders.containsKey("maxEmbeddedResources")) {
- maxEmbeddedResources = Integer.parseInt(httpHeaders.getFirst("maxEmbeddedResources"));
- }
- return new HandlerConfig(BasicContentHandlerFactory.parseHandlerType(handlerTypeName, DEFAULT_HANDLER_TYPE), parseMode, writeLimit, maxEmbeddedResources,
- TikaResource.getThrowOnWriteLimitReached(httpHeaders));
- }
-
- /**
- * Returns an InputStream that can be deserialized as a list of
- * {@link Metadata} objects.
- * The first in the list represents the main document, and the
- * rest represent metadata for the embedded objects. This works
- * recursively through all descendants of the main document, not
- * just the immediate children.
- * <p>
- * The extracted text content is stored with the key
- * {@link org.apache.tika.metadata.TikaCoreProperties#TIKA_CONTENT}.
- * <p>
- * Specify the handler for the content (xml, html, text, ignore)
- * in the path:<br/>
- * /rmeta/form (default: xml)<br/>
- * /rmeta/form/xml (store the content as xml)<br/>
- * /rmeta/form/text (store the content as text)<br/>
- * /rmeta/form/ignore (don't record any content)<br/>
- *
- * @param att attachment
- * @param info uri info
- * @param handlerTypeName which type of handler to use
- * @return InputStream that can be deserialized as a list of {@link Metadata} objects
- * @throws Exception
- */
- @POST
- @Consumes("multipart/form-data")
- @Produces({"application/json"})
- @Path("form{" + HANDLER_TYPE_PARAM + " : (\\w+)?}")
- public Response getMetadataFromMultipart(Attachment att, @Context UriInfo info, @PathParam(HANDLER_TYPE_PARAM) String handlerTypeName) throws Exception {
- return Response
- .ok(parseMetadataToMetadataList(att.getObject(InputStream.class), new Metadata(), att.getHeaders(), info,
- buildHandlerConfig(att.getHeaders(), handlerTypeName, HandlerConfig.PARSE_MODE.RMETA)))
- .build();
- }
-
- /**
- * Returns an InputStream that can be deserialized as a list of
- * {@link Metadata} objects.
- * The first in the list represents the main document, and the
- * rest represent metadata for the embedded objects. This works
- * recursively through all descendants of the main document, not
- * just the immediate children.
- * <p>
- * The extracted text content is stored with the key
- * {@link org.apache.tika.metadata.TikaCoreProperties#TIKA_CONTENT}.
- * <p>
- * Specify the handler for the content (xml, html, text, ignore)
- * in the path:<br/>
- * /rmeta (default: xml)<br/>
- * /rmeta/xml (store the content as xml)<br/>
- * /rmeta/text (store the content as text)<br/>
- * /rmeta/ignore (don't record any content)<br/>
- *
- * @param info uri info
- * @param handlerTypeName which type of handler to use
- * @return InputStream that can be deserialized as a list of {@link Metadata} objects
- * @throws Exception
- */
-
- @PUT
- @Produces("application/json")
- @Path("{" + HANDLER_TYPE_PARAM + " : (\\w+)?}")
- public Response getMetadata(InputStream is, @Context HttpHeaders httpHeaders, @Context UriInfo info, @PathParam(HANDLER_TYPE_PARAM) String handlerTypeName) throws Exception {
- Metadata metadata = new Metadata();
- return Response
- .ok(parseMetadataToMetadataList(TikaResource.getInputStream(is, metadata, httpHeaders, info), metadata, httpHeaders.getRequestHeaders(), info,
- buildHandlerConfig(httpHeaders.getRequestHeaders(), handlerTypeName, HandlerConfig.PARSE_MODE.RMETA)))
- .build();
- }
-
- private MetadataList parseMetadataToMetadataList(InputStream is, Metadata metadata, MultivaluedMap<String, String> httpHeaders, UriInfo info, HandlerConfig handlerConfig)
- throws Exception {
- return new MetadataList(parseMetadata(is, metadata, httpHeaders, info, handlerConfig));
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaDetectors.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaDetectors.java
deleted file mode 100644
index d9166dca3..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaDetectors.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core.resource;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import jakarta.ws.rs.GET;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.Produces;
-
-import org.apache.tika.detect.CompositeDetector;
-import org.apache.tika.detect.Detector;
-import org.apache.tika.server.core.HTMLHelper;
-
-/**
- * <p>Provides details of all the {@link Detector}s registered with
- * Apache Tika, similar to <em>--list-detectors</em> with the Tika CLI.
- */
-@Path("/detectors")
-public class TikaDetectors {
-
- private HTMLHelper html;
-
- public TikaDetectors() {
- this.html = new HTMLHelper();
- }
-
- @GET
- @Produces("text/html")
- public String getDectorsHTML() {
- StringBuffer h = new StringBuffer();
- html.generateHeader(h, "Detectors available to Apache Tika");
- detectorAsHTML(TikaResource
- .getConfig()
- .getDetector(), h, 2);
- html.generateFooter(h);
- return h.toString();
- }
-
- private void detectorAsHTML(Detector d, StringBuffer html, int level) {
- html.append("<h");
- html.append(level);
- html.append(">");
- String name = d
- .getClass()
- .getName();
- html.append(name.substring(name.lastIndexOf('.') + 1));
- html.append("</h");
- html.append(level);
- html.append(">");
- html.append("<p>Class: ");
- html.append(name);
- html.append("</p>");
- if (d instanceof CompositeDetector) {
- html.append("<p>Composite Detector</p>");
- for (Detector cd : ((CompositeDetector) d).getDetectors()) {
- detectorAsHTML(cd, html, level + 1);
- }
- }
- }
-
- @GET
- @Produces(jakarta.ws.rs.core.MediaType.APPLICATION_JSON)
- public String getDetectorsJSON() throws IOException {
- Map<String, Object> details = new HashMap<>();
- detectorAsMap(TikaResource
- .getConfig()
- .getDetector(), details);
- return new ObjectMapper().writeValueAsString(details);
- }
-
- private void detectorAsMap(Detector d, Map<String, Object> details) {
- details.put("name", d
- .getClass()
- .getName());
-
- boolean isComposite = (d instanceof CompositeDetector);
- details.put("composite", isComposite);
- if (isComposite) {
- List<Map<String, Object>> c = new ArrayList<>();
- for (Detector cd : ((CompositeDetector) d).getDetectors()) {
- Map<String, Object> cdet = new HashMap<>();
- detectorAsMap(cd, cdet);
- c.add(cdet);
- }
- details.put("children", c);
- }
- }
-
- @GET
- @Produces("text/plain")
- public String getDetectorsPlain() {
- StringBuffer text = new StringBuffer();
- renderDetector(TikaResource
- .getConfig()
- .getDetector(), text, 0);
- return text.toString();
- }
-
- private void renderDetector(Detector d, StringBuffer text, int indent) {
- boolean isComposite = (d instanceof CompositeDetector);
- String name = d
- .getClass()
- .getName();
-
- for (int i = 0; i < indent; i++) {
- text.append(" ");
- }
- text.append(name);
- if (isComposite) {
- text.append(" (Composite Detector):\n");
-
- List<Detector> subDetectors = ((CompositeDetector) d).getDetectors();
- for (Detector sd : subDetectors) {
- renderDetector(sd, text, indent + 1);
- }
- } else {
- text.append("\n");
- }
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaMimeTypes.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaMimeTypes.java
deleted file mode 100644
index 36f74418b..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaMimeTypes.java
+++ /dev/null
@@ -1,418 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core.resource;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import jakarta.ws.rs.GET;
-import jakarta.ws.rs.NotFoundException;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.PathParam;
-import jakarta.ws.rs.Produces;
-
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.mime.MediaTypeRegistry;
-import org.apache.tika.mime.MimeType;
-import org.apache.tika.mime.MimeTypeException;
-import org.apache.tika.mime.MimeTypes;
-import org.apache.tika.parser.CompositeParser;
-import org.apache.tika.parser.Parser;
-import org.apache.tika.server.core.HTMLHelper;
-
-/**
- * <p>Provides details of all the mimetypes known to Apache Tika,
- * similar to <em>--list-supported-types</em> with the Tika CLI.
- * <p>Can also provide full details on a single known type.
- */
-@Path("/mime-types")
-public class TikaMimeTypes {
-
- private HTMLHelper html;
-
- public TikaMimeTypes() {
- this.html = new HTMLHelper();
- }
-
- private static String[] copyToStringArray(MediaType[] aliases) {
- String[] strings = new String[aliases.length];
- for (int i = 0; i < aliases.length; i++) {
- strings[i] = aliases[i].toString();
- }
- return strings;
- }
-
- @GET
- @Produces("text/html")
- public String getMimeTypesHTML() {
- StringBuffer h = new StringBuffer();
- html.generateHeader(h, "Apache Tika Supported Mime Types");
-
- // Get our types
- List<MediaTypeDetails> types = getMediaTypes();
-
- // Get the first type in each section
- SortedMap<String, String> firstType = new TreeMap<>();
- for (MediaTypeDetails type : types) {
- if (!firstType.containsKey(type.type.getType())) {
- firstType.put(type.type.getType(), type.type.toString());
- }
- }
- h.append("<ul>");
- for (String section : firstType.keySet()) {
- h
- .append("<li><a href=\"#")
- .append(firstType.get(section))
- .append("\">")
- .append(section)
- .append("</a></li>\n");
- }
- h.append("</ul>");
-
- // Output all of them
- for (MediaTypeDetails type : types) {
- h
- .append("<a name=\"")
- .append(type.type)
- .append("\"></a>\n");
- h
- .append("<h2><a href=\"mime-types/")
- .append(type.type)
- .append("\">")
- .append(type.type)
- .append("</a></h2>\n");
-
- for (MediaType alias : type.aliases) {
- h
- .append("<div>Alias: ")
- .append(alias)
- .append("</div>\n");
- }
- if (type.supertype != null) {
- h
- .append("<div>Super Type: <a href=\"#")
- .append(type.supertype)
- .append("\">")
- .append(type.supertype)
- .append("</a></div>\n");
- }
- if (type.mime != null) {
- if (!type.mime
- .getDescription()
- .isEmpty()) {
- h
- .append("<div>Description: ")
- .append(type.mime.getDescription())
- .append("</div>\n");
- }
- if (!type.mime
- .getAcronym()
- .isEmpty()) {
- h
- .append("<div>Acronym: ")
- .append(type.mime.getAcronym())
- .append("</div>\n");
- }
- if (!type.mime
- .getExtension()
- .isEmpty()) {
- h
- .append("<div>Default Extension: ")
- .append(type.mime.getExtension())
- .append("</div>\n");
- }
- }
-
- if (type.parser != null) {
- h
- .append("<div>Parser: ")
- .append(type.parser)
- .append("</div>\n");
- }
- }
-
- html.generateFooter(h);
- return h.toString();
- }
-
- @GET
- @Path("/{type}/{subtype}")
- @Produces("text/html")
- public String getMimeTypeDetailsHTML(@PathParam("type") String typePart, @PathParam("subtype") String subtype) {
- MediaTypeDetails type = getMediaType(typePart, subtype);
-
- StringBuffer h = new StringBuffer();
- html.generateHeader(h, "Apache Tika Details on Mime Type " + type.type);
- h
- .append("<h2>")
- .append(type.type)
- .append("</h2>\n");
-
- for (MediaType alias : type.aliases) {
- h
- .append("<div>Alias: ")
- .append(alias)
- .append("</div>\n");
- }
- if (type.supertype != null) {
- h
- .append("<div>Super Type: <a href=\"#")
- .append(type.supertype)
- .append("\">")
- .append(type.supertype)
- .append("</a></div>\n");
- }
- if (type.mime != null) {
- if (!type.mime
- .getDescription()
- .isEmpty()) {
- h
- .append("<div>Description: ")
- .append(type.mime.getDescription())
- .append("</div>\n");
- }
- if (!type.mime
- .getAcronym()
- .isEmpty()) {
- h
- .append("<div>Acronym: ")
- .append(type.mime.getAcronym())
- .append("</div>\n");
- }
- if (!type.mime
- .getUniformTypeIdentifier()
- .isEmpty()) {
- h
- .append("<div>Uniform Type Identifier: ")
- .append(type.mime.getUniformTypeIdentifier())
- .append("</div>\n");
- }
- for (URI uri : type.mime.getLinks()) {
- h
- .append("<div>Link: ")
- .append(uri)
- .append("</div>\n");
- }
- if (!type.mime
- .getExtension()
- .isEmpty()) {
- h
- .append("<div>Default Extension: ")
- .append(type.mime.getExtension())
- .append("</div>\n");
- }
- for (String ext : type.mime.getExtensions()) {
- h
- .append("<div>Extension: ")
- .append(ext)
- .append("</div>\n");
- }
- }
-
- if (type.parser != null) {
- h
- .append("<div>Parser: ")
- .append(type.parser)
- .append("</div>\n");
- }
-
- html.generateFooter(h);
- return h.toString();
- }
-
- @GET
- @Produces(jakarta.ws.rs.core.MediaType.APPLICATION_JSON)
- public String getMimeTypesJSON() throws IOException {
- Map<String, Object> details = new HashMap<>();
-
- for (MediaTypeDetails type : getMediaTypes()) {
- Map<String, Object> typeDets = new HashMap<>();
-
- typeDets.put("alias", copyToStringArray(type.aliases));
- if (type.supertype != null) {
- typeDets.put("supertype", type.supertype.toString());
- }
- if (type.parser != null) {
- typeDets.put("parser", type.parser);
- }
-
- details.put(type.type.toString(), typeDets);
- }
-
- return new ObjectMapper()
- .writerWithDefaultPrettyPrinter()
- .writeValueAsString(details);
- }
-
- @GET
- @Path("/{type}/{subtype}")
- @Produces(jakarta.ws.rs.core.MediaType.APPLICATION_JSON)
- public String getMimeTypeDetailsJSON(@PathParam("type") String typePart, @PathParam("subtype") String subtype) throws IOException {
- MediaTypeDetails type = getMediaType(typePart, subtype);
- Map<String, Object> details = new HashMap<>();
-
- details.put("type", type.type.toString());
- details.put("alias", copyToStringArray(type.aliases));
- if (type.supertype != null) {
- details.put("supertype", type.supertype.toString());
- }
- if (type.parser != null) {
- details.put("parser", type.parser);
- }
- if (type.mime != null) {
- if (!type.mime
- .getDescription()
- .isEmpty()) {
- details.put("description", type.mime.getDescription());
- }
- if (!type.mime
- .getAcronym()
- .isEmpty()) {
- details.put("acronym", type.mime.getAcronym());
- }
- if (!type.mime
- .getUniformTypeIdentifier()
- .isEmpty()) {
- details.put("uniformTypeIdentifier", type.mime.getUniformTypeIdentifier());
- }
- if (!type.mime
- .getLinks()
- .isEmpty()) {
- details.put("links", type.mime.getLinks());
- }
- if (!type.mime
- .getExtension()
- .isEmpty()) {
- details.put("defaultExtension", type.mime.getExtension());
- }
- if (!type.mime
- .getExtensions()
- .isEmpty()) {
- details.put("extensions", type.mime.getExtensions());
- }
- }
-
- return new ObjectMapper()
- .writerWithDefaultPrettyPrinter()
- .writeValueAsString(details);
- }
-
- @GET
- @Produces("text/plain")
- public String getMimeTypesPlain() {
- StringBuilder text = new StringBuilder();
-
- for (MediaTypeDetails type : getMediaTypes()) {
- text.append(type.type.toString());
- text.append("\n");
-
- for (MediaType alias : type.aliases) {
- text
- .append(" alias: ")
- .append(alias)
- .append("\n");
- }
- if (type.supertype != null) {
- text
- .append(" supertype: ")
- .append(type.supertype.toString())
- .append("\n");
- }
-
- if (type.parser != null) {
- text
- .append(" parser: ")
- .append(type.parser)
- .append("\n");
- }
- }
-
- return text.toString();
- }
-
- protected MediaTypeDetails getMediaType(String type, String subtype) throws NotFoundException {
- MediaType mt = MediaType.parse(type + "/" + subtype);
- for (MediaTypeDetails mtd : getMediaTypes()) {
- if (mtd.type.equals(mt)) {
- return mtd;
- }
- }
- throw new NotFoundException("No Media Type registered in Tika for " + mt);
- }
-
- protected List<MediaTypeDetails> getMediaTypes() {
- TikaConfig config = TikaResource.getConfig();
- MimeTypes mimeTypes = config.getMimeRepository();
- MediaTypeRegistry registry = config.getMediaTypeRegistry();
- Map<MediaType, Parser> parsers = ((CompositeParser) config.getParser()).getParsers();
-
- List<MediaTypeDetails> types = new ArrayList<>(registry
- .getTypes()
- .size());
-
- for (MediaType type : registry.getTypes()) {
- MediaTypeDetails details = new MediaTypeDetails();
- details.type = type;
- details.aliases = registry
- .getAliases(type)
- .toArray(new MediaType[0]);
-
- try {
- details.mime = mimeTypes.getRegisteredMimeType(type.toString());
- } catch (MimeTypeException e) {
- // Ignore if invalid
- }
-
- MediaType supertype = registry.getSupertype(type);
- if (supertype != null && !MediaType.OCTET_STREAM.equals(supertype)) {
- details.supertype = supertype;
- }
-
- Parser p = parsers.get(type);
- if (p != null) {
- if (p instanceof CompositeParser) {
- p = ((CompositeParser) p)
- .getParsers()
- .get(type);
- }
- details.parser = p
- .getClass()
- .getName();
- }
-
- types.add(details);
- }
-
- return types;
- }
-
- private static class MediaTypeDetails {
- private MediaType type;
- private MediaType[] aliases;
- private MediaType supertype;
- private MimeType mime;
- private String parser;
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaParsers.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaParsers.java
deleted file mode 100644
index 64e645c59..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaParsers.java
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core.resource;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import jakarta.ws.rs.GET;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.Produces;
-
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.parser.CompositeParser;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.apache.tika.parser.ParserDecorator;
-import org.apache.tika.server.core.HTMLHelper;
-
-/**
- * <p>Provides details of all the {@link Parser}s registered with
- * Apache Tika, similar to <em>--list-parsers</em> and
- * <em>--list-parser-details</em> within the Tika CLI.
- */
-@Path("/parsers")
-public class TikaParsers {
- private static final ParseContext EMPTY_PC = new ParseContext();
- private HTMLHelper html;
-
- public TikaParsers() {
- this.html = new HTMLHelper();
- }
-
- @GET
- @Path("/details")
- @Produces("text/html")
- public String getParserDetailsHTML() {
- return getParsersHTML(true);
- }
-
- @GET
- @Produces("text/html")
- public String getParsersHTML() {
- return getParsersHTML(false);
- }
-
- protected String getParsersHTML(boolean withMimeTypes) {
- ParserDetails p = new ParserDetails(TikaResource
- .getConfig()
- .getParser());
-
- StringBuffer h = new StringBuffer();
- html.generateHeader(h, "Parsers available to Apache Tika");
- parserAsHTML(p, withMimeTypes, h, 2);
- html.generateFooter(h);
- return h.toString();
- }
-
- private void parserAsHTML(ParserDetails p, boolean withMimeTypes, StringBuffer html, int level) {
- html.append("<h");
- html.append(level);
- html.append(">");
- html.append(p.shortName);
- html.append("</h");
- html.append(level);
- html.append(">");
- html.append("<p>Class: ");
- html.append(p.className);
- html.append("</p>");
- if (p.isDecorated) {
- html.append("<p>Decorated Parser");
- if (p.decoratedBy != null) {
- html
- .append(" - ")
- .append(p.decoratedBy);
- }
- html.append("</p>");
- }
- if (p.isComposite) {
- html.append("<p>Composite Parser</p>");
- html.append("<div style=\"margin-left: 1em\">\n");
- for (Parser cp : p.childParsers) {
- parserAsHTML(new ParserDetails(cp), withMimeTypes, html, level + 1);
- }
- html.append("</div>\n");
- } else if (withMimeTypes) {
- html.append("<p>Mime Types:");
- html.append("<ul>");
- for (MediaType mt : p.supportedTypes) {
- html.append("<li>");
- html.append(mt.toString());
- html.append("</li>");
- }
- html.append("</ul>");
- html.append("</p>");
- }
- html.append("\n");
- }
-
- @GET
- @Path("/details")
- @Produces(jakarta.ws.rs.core.MediaType.APPLICATION_JSON)
- public String getParserDetailsJSON() throws IOException {
- return getParsersJSON(true);
- }
-
- @GET
- @Produces(jakarta.ws.rs.core.MediaType.APPLICATION_JSON)
- public String getParsersJSON() throws IOException {
- return getParsersJSON(false);
- }
-
- protected String getParsersJSON(boolean withMimeTypes) throws IOException {
- Map<String, Object> details = new HashMap<>();
- parserAsMap(new ParserDetails(TikaResource
- .getConfig()
- .getParser()), withMimeTypes, details);
- ObjectMapper objectMapper = new ObjectMapper();
- return objectMapper
- .writerWithDefaultPrettyPrinter()
- .writeValueAsString(details);
- }
-
- private void parserAsMap(ParserDetails p, boolean withMimeTypes, Map<String, Object> details) {
- details.put("name", p.className);
- details.put("composite", p.isComposite);
- details.put("decorated", p.isDecorated);
-
- if (p.isComposite) {
- List<Map<String, Object>> c = new ArrayList<>();
- for (Parser cp : p.childParsers) {
- Map<String, Object> cdet = new HashMap<>();
- parserAsMap(new ParserDetails(cp), withMimeTypes, cdet);
- c.add(cdet);
- }
- details.put("children", c);
- } else if (withMimeTypes) {
- List<String> mts = new ArrayList<>(p.supportedTypes.size());
- for (MediaType mt : p.supportedTypes) {
- mts.add(mt.toString());
- }
- details.put("supportedTypes", mts);
- }
- }
-
- @GET
- @Path("/details")
- @Produces("text/plain")
- public String getParserDetailssPlain() {
- return getParsersPlain(true);
- }
-
- @GET
- @Produces("text/plain")
- public String getParsersPlain() {
- return getParsersPlain(false);
- }
-
- protected String getParsersPlain(boolean withMimeTypes) {
- StringBuffer text = new StringBuffer();
- renderParser(new ParserDetails(TikaResource
- .getConfig()
- .getParser()), withMimeTypes, text, "");
- return text.toString();
- }
-
- private void renderParser(ParserDetails p, boolean withMimeTypes, StringBuffer text, String indent) {
- String nextIndent = indent + " ";
-
- text.append(indent);
- text.append(p.className);
- if (p.isDecorated) {
- text.append(" (Decorated Parser");
- if (p.decoratedBy != null) {
- text
- .append(" ")
- .append(p.decoratedBy);
- }
- text.append(")");
- }
- if (p.isComposite) {
- text.append(" (Composite Parser):\n");
-
- for (Parser cp : p.childParsers) {
- renderParser(new ParserDetails(cp), withMimeTypes, text, nextIndent);
- }
- } else {
- text.append("\n");
- if (withMimeTypes) {
- for (MediaType mt : p.supportedTypes) {
- text.append(nextIndent);
- text.append("Supports: ");
- text.append(mt.toString());
- text.append("\n");
- }
- }
- }
- }
-
- private static class ParserDetails {
- private String className;
- private String shortName;
- private boolean isComposite;
- private boolean isDecorated;
- private String decoratedBy;
- private Set<MediaType> supportedTypes;
- private List<Parser> childParsers;
-
- private ParserDetails(Parser p) {
- if (p instanceof ParserDecorator) {
- isDecorated = true;
- decoratedBy = ((ParserDecorator) p).getDecorationName();
- p = ((ParserDecorator) p).getWrappedParser();
- }
-
- className = p
- .getClass()
- .getName();
- shortName = className.substring(className.lastIndexOf('.') + 1);
-
- if (p instanceof CompositeParser) {
- isComposite = true;
- supportedTypes = Collections.emptySet();
-
- // Get the unique set of child parsers
- Set<Parser> children = new HashSet<>(((CompositeParser) p)
- .getParsers(EMPTY_PC)
- .values());
- // Sort it by class name
- childParsers = new ArrayList<>(children);
- childParsers.sort(Comparator.comparing(parser -> parser
- .getClass()
- .getName()));
- } else {
- supportedTypes = p.getSupportedTypes(EMPTY_PC);
- }
- }
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
deleted file mode 100644
index 7fc042a7d..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
+++ /dev/null
@@ -1,684 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core.resource;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.tika.server.core.resource.RecursiveMetadataResource.DEFAULT_HANDLER_TYPE;
-import static org.apache.tika.server.core.resource.RecursiveMetadataResource.HANDLER_TYPE_PARAM;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.TransformerConfigurationException;
-import javax.xml.transform.sax.SAXTransformerFactory;
-import javax.xml.transform.sax.TransformerHandler;
-import javax.xml.transform.stream.StreamResult;
-
-import jakarta.ws.rs.Consumes;
-import jakarta.ws.rs.GET;
-import jakarta.ws.rs.POST;
-import jakarta.ws.rs.PUT;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.PathParam;
-import jakarta.ws.rs.Produces;
-import jakarta.ws.rs.WebApplicationException;
-import jakarta.ws.rs.core.Context;
-import jakarta.ws.rs.core.HttpHeaders;
-import jakarta.ws.rs.core.MultivaluedMap;
-import jakarta.ws.rs.core.Response;
-import jakarta.ws.rs.core.StreamingOutput;
-import jakarta.ws.rs.core.UriInfo;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.Strings;
-import org.apache.cxf.attachment.ContentDisposition;
-import org.apache.cxf.jaxrs.ext.multipart.Attachment;
-import org.apache.cxf.jaxrs.impl.MetadataMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.Tika;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.config.TikaTaskTimeout;
-import org.apache.tika.exception.EncryptedDocumentException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.exception.WriteLimitReachedException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.AutoDetectParser;
-import org.apache.tika.parser.DigestingParser;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.apache.tika.pipes.core.HandlerConfig;
-import org.apache.tika.sax.BasicContentHandlerFactory;
-import org.apache.tika.sax.BodyContentHandler;
-import org.apache.tika.sax.ExpandedTitleContentHandler;
-import org.apache.tika.sax.RichTextContentHandler;
-import org.apache.tika.sax.boilerpipe.BoilerpipeContentHandler;
-import org.apache.tika.server.core.CompositeParseContextConfig;
-import org.apache.tika.server.core.InputStreamFactory;
-import org.apache.tika.server.core.ParseContextConfig;
-import org.apache.tika.server.core.ServerStatus;
-import org.apache.tika.server.core.TikaServerConfig;
-import org.apache.tika.server.core.TikaServerParseException;
-import org.apache.tika.utils.ExceptionUtils;
-import org.apache.tika.utils.XMLReaderUtils;
-
-@Path("/tika")
-public class TikaResource {
-
- public static final String GREETING = "This is Tika Server (" + Tika.getString() + "). Please PUT\n";
- private static final String META_PREFIX = "meta_";
- private static final Logger LOG = LoggerFactory.getLogger(TikaResource.class);
- private static Pattern ALLOWABLE_HEADER_CHARS = Pattern.compile("(?i)^[-/_+\\.A-Z0-9 ]+$");
- private static TikaConfig TIKA_CONFIG;
- private static TikaServerConfig TIKA_SERVER_CONFIG;
- private static DigestingParser.Digester DIGESTER = null;
- private static InputStreamFactory INPUTSTREAM_FACTORY = null;
- private static ServerStatus SERVER_STATUS = null;
-
- private static ParseContextConfig PARSE_CONTEXT_CONFIG = new CompositeParseContextConfig();
-
-
- public static void init(TikaConfig config, TikaServerConfig tikaServerConfg, DigestingParser.Digester digester, InputStreamFactory inputStreamFactory,
- ServerStatus serverStatus) {
- TIKA_CONFIG = config;
- TIKA_SERVER_CONFIG = tikaServerConfg;
- DIGESTER = digester;
- INPUTSTREAM_FACTORY = inputStreamFactory;
- SERVER_STATUS = serverStatus;
- }
-
-
- @SuppressWarnings("serial")
- public static Parser createParser() {
- final Parser parser = new AutoDetectParser(TIKA_CONFIG);
-
- if (DIGESTER != null) {
- boolean skipContainer = false;
- if (TIKA_CONFIG
- .getAutoDetectParserConfig()
- .getDigesterFactory() != null && TIKA_CONFIG
- .getAutoDetectParserConfig()
- .getDigesterFactory()
- .isSkipContainerDocument()) {
- skipContainer = true;
- }
- return new DigestingParser(parser, DIGESTER, skipContainer);
- }
- return parser;
- }
-
- public static TikaConfig getConfig() {
- return TIKA_CONFIG;
- }
-
- public static String detectFilename(MultivaluedMap<String, String> httpHeaders) {
-
- String disposition = httpHeaders.getFirst("Content-Disposition");
- if (disposition != null) {
- ContentDisposition c = new ContentDisposition(disposition);
-
- // only support "attachment" dispositions
- if ("attachment".equals(c.getType())) {
- String fn = c.getParameter("filename");
- if (fn != null) {
- return fn;
- }
- }
- }
-
- // this really should not be used, since it's not an official field
- return httpHeaders.getFirst("File-Name");
- }
-
- public static void fillParseContext(MultivaluedMap<String, String> httpHeaders, Metadata metadata, ParseContext parseContext) {
- PARSE_CONTEXT_CONFIG.configure(httpHeaders, metadata, parseContext);
- }
-
- public static InputStream getInputStream(InputStream is, Metadata metadata, HttpHeaders headers, UriInfo uriInfo) {
- try {
- return INPUTSTREAM_FACTORY.getInputStream(is, metadata, headers, uriInfo);
- } catch (IOException e) {
- throw new TikaServerParseException(e);
- }
- }
-
- /**
- * Utility method to set a property on a class via reflection.
- *
- * @param object the <code>Object</code> to set the property on.
- * @param key the key of the HTTP Header.
- * @param val the value of HTTP header.
- * @param prefix the name of the HTTP Header prefix used to find property.
- * @throws WebApplicationException thrown when field cannot be found.
- */
- public static void processHeaderConfig(Object object, String key, String val, String prefix) {
- try {
- String property = Strings.CI.removeStart(key, prefix);
- Field field = null;
- try {
- field = object
- .getClass()
- .getDeclaredField(StringUtils.uncapitalize(property));
- } catch (NoSuchFieldException e) {
- // try to match field case-insensitive way
- for (Field aField : object
- .getClass()
- .getDeclaredFields()) {
- if (aField
- .getName()
- .equalsIgnoreCase(property)) {
- field = aField;
- break;
- }
- }
- }
- String setter = field != null ? field.getName() : property;
- setter = "set" + setter
- .substring(0, 1)
- .toUpperCase(Locale.US) + setter.substring(1);
- //default assume string class
- //if there's a more specific type, e.g. double, int, boolean
- //try that.
- Class clazz = String.class;
- if (field != null) {
- if (field.getType() == int.class || field.getType() == Integer.class) {
- clazz = int.class;
- } else if (field.getType() == double.class) {
- clazz = double.class;
- } else if (field.getType() == Double.class) {
- clazz = Double.class;
- } else if (field.getType() == float.class) {
- clazz = float.class;
- } else if (field.getType() == Float.class) {
- clazz = Float.class;
- } else if (field.getType() == boolean.class) {
- clazz = boolean.class;
- } else if (field.getType() == Boolean.class) {
- clazz = Boolean.class;
- } else if (field.getType() == long.class) {
- clazz = long.class;
- } else if (field.getType() == Long.class) {
- clazz = Long.class;
- }
- }
-
- Method m = tryToGetMethod(object, setter, clazz);
- //if you couldn't find more specific setter, back off
- //to string setter and try that.
- if (m == null && clazz != String.class) {
- m = tryToGetMethod(object, setter, String.class);
- }
-
- if (m != null) {
- if (clazz == String.class) {
- checkTrustWorthy(setter, val);
- m.invoke(object, val);
- } else if (clazz == int.class || clazz == Integer.class) {
- m.invoke(object, Integer.parseInt(val));
- } else if (clazz == double.class || clazz == Double.class) {
- m.invoke(object, Double.parseDouble(val));
- } else if (clazz == boolean.class || clazz == Boolean.class) {
- m.invoke(object, Boolean.parseBoolean(val));
- } else if (clazz == float.class || clazz == Float.class) {
- m.invoke(object, Float.parseFloat(val));
- } else if (clazz == long.class || clazz == Long.class) {
- m.invoke(object, Long.parseLong(val));
- } else {
- throw new IllegalArgumentException("setter must be String, int, float, double or boolean...for now");
- }
- } else {
- throw new NoSuchMethodException("Couldn't find: " + setter);
- }
-
- } catch (Throwable ex) {
- // TIKA-3345
- String error = (!(ex.getCause() instanceof IllegalArgumentException)) ? String.format(Locale.ROOT, "%s is an invalid %s header", key, prefix) :
- String.format(Locale.ROOT, "%s is an invalid %s header value", val, key);
- throw new WebApplicationException(error, Response.Status.BAD_REQUEST);
- }
- }
-
- private static void checkTrustWorthy(String setter, String val) {
- if (setter == null || val == null) {
- throw new IllegalArgumentException("setter and val must not be null");
- }
- if (setter
- .toLowerCase(Locale.US)
- .contains("trusted")) {
- throw new IllegalArgumentException("Can't call a trusted method via tika-server headers");
- }
- Matcher m = ALLOWABLE_HEADER_CHARS.matcher(val);
- if (!m.find()) {
- throw new IllegalArgumentException("Header val: " + val + " contains illegal characters. " + "Must contain: TikaResource.ALLOWABLE_HEADER_CHARS");
- }
- }
-
- /**
- * Tries to get method. Silently swallows NoMethodException and returns
- * <code>null</code> if not found.
- *
- * @param object the object to get method from.
- * @param method the name of the method to get.
- * @param clazz the parameter type of the method to get.
- * @return the found method instance
- */
- private static Method tryToGetMethod(Object object, String method, Class clazz) {
- try {
- return object
- .getClass()
- .getMethod(method, clazz);
- } catch (NoSuchMethodException e) {
- return null;
- }
- }
-
- @SuppressWarnings("serial")
- public static void fillMetadata(Parser parser, Metadata metadata, MultivaluedMap<String, String> httpHeaders) {
- String fileName = detectFilename(httpHeaders);
- if (fileName != null) {
- metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, fileName);
- }
-
- String contentTypeHeader = httpHeaders.getFirst(HttpHeaders.CONTENT_TYPE);
- jakarta.ws.rs.core.MediaType mediaType = (contentTypeHeader == null || "*/*".equals(contentTypeHeader)) ? null : jakarta.ws.rs.core.MediaType.valueOf(contentTypeHeader);
- if (mediaType != null && "xml".equals(mediaType.getSubtype())) {
- mediaType = null;
- }
-
- if (mediaType != null && mediaType.equals(jakarta.ws.rs.core.MediaType.APPLICATION_OCTET_STREAM_TYPE)) {
- mediaType = null;
- }
-
- if (mediaType != null) {
- metadata.add(Metadata.CONTENT_TYPE, mediaType.toString());
- metadata.add(TikaCoreProperties.CONTENT_TYPE_USER_OVERRIDE, mediaType.toString());
- }
-
- if (httpHeaders.containsKey("Content-Length")) {
- metadata.set(Metadata.CONTENT_LENGTH, httpHeaders.getFirst("Content-Length"));
- }
-
- for (Map.Entry<String, List<String>> e : httpHeaders.entrySet()) {
- if (e
- .getKey()
- .startsWith(META_PREFIX)) {
- String tikaKey = e
- .getKey()
- .substring(META_PREFIX.length());
- for (String value : e.getValue()) {
- metadata.add(tikaKey, value);
- }
- }
- }
- }
-
- /**
- * Use this to call a parser and unify exception handling.
- * NOTE: This call to parse closes the InputStream. DO NOT surround
- * the call in an auto-close block.
- *
- * @param parser parser to use
- * @param logger logger to use
- * @param path file path
- * @param inputStream inputStream (which is closed by this call!)
- * @param handler handler to use
- * @param metadata metadata
- * @param parseContext parse context
- * @throws IOException wrapper for all exceptions
- */
- public static void parse(Parser parser, Logger logger, String path, InputStream inputStream, ContentHandler handler, Metadata metadata, ParseContext parseContext)
- throws IOException {
-
- checkIsOperating();
- String fileName = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
- long timeoutMillis = getTaskTimeout(parseContext);
-
- long taskId = SERVER_STATUS.start(ServerStatus.TASK.PARSE, fileName, timeoutMillis);
- try {
- parser.parse(inputStream, handler, metadata, parseContext);
- } catch (SAXException e) {
- throw new TikaServerParseException(e);
- } catch (EncryptedDocumentException e) {
- logger.warn("{}: Encrypted document ({})", path, fileName, e);
- throw new TikaServerParseException(e);
- } catch (Exception e) {
- if (!WriteLimitReachedException.isWriteLimitReached(e)) {
- logger.warn("{}: Text extraction failed ({})", path, fileName, e);
- }
- throw new TikaServerParseException(e);
- } catch (OutOfMemoryError e) {
- logger.warn("{}: OOM ({})", path, fileName, e);
- SERVER_STATUS.setStatus(ServerStatus.STATUS.ERROR);
- throw e;
- } finally {
- SERVER_STATUS.complete(taskId);
- inputStream.close();
- }
- }
-
- protected static long getTaskTimeout(ParseContext parseContext) {
-
- TikaTaskTimeout tikaTaskTimeout = parseContext.get(TikaTaskTimeout.class);
- long timeoutMillis = TIKA_SERVER_CONFIG.getTaskTimeoutMillis();
-
- if (tikaTaskTimeout != null) {
- if (tikaTaskTimeout.getTimeoutMillis() > TIKA_SERVER_CONFIG.getTaskTimeoutMillis()) {
- throw new IllegalArgumentException(
- "Can't request a timeout ( " + tikaTaskTimeout.getTimeoutMillis() + "ms) greater than the taskTimeoutMillis set in the server config (" +
- TIKA_SERVER_CONFIG.getTaskTimeoutMillis() + "ms)");
- }
- timeoutMillis = tikaTaskTimeout.getTimeoutMillis();
- if (timeoutMillis < TIKA_SERVER_CONFIG.getMinimumTimeoutMillis()) {
- throw new WebApplicationException(new IllegalArgumentException(
- "taskTimeoutMillis must be > " + "minimumTimeoutMillis, currently set to (" + TIKA_SERVER_CONFIG.getMinimumTimeoutMillis() + "ms)"),
- Response.Status.BAD_REQUEST);
- }
- }
- return timeoutMillis;
- }
-
- public static void checkIsOperating() {
- //check that server is not in shutdown mode
- if (!SERVER_STATUS.isOperating()) {
- throw new WebApplicationException(Response.Status.SERVICE_UNAVAILABLE);
- }
- }
-
- public static void logRequest(Logger logger, String endpoint, Metadata metadata) {
-
- if (metadata.get(org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE) == null) {
- logger.info("{} (autodetecting type)", endpoint);
- } else {
- logger.info("{} ({})", endpoint, metadata.get(org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE));
- }
- }
-
- public static boolean getThrowOnWriteLimitReached(MultivaluedMap<String, String> httpHeaders) {
- if (httpHeaders.containsKey("throwOnWriteLimitReached")) {
- String val = httpHeaders.getFirst("throwOnWriteLimitReached");
- if ("true".equalsIgnoreCase(val)) {
- return true;
- } else if ("false".equalsIgnoreCase(val)) {
- return false;
- } else {
- throw new IllegalArgumentException("'throwOnWriteLimitReached' must be either 'true' or 'false'");
- }
- }
- return HandlerConfig.DEFAULT_HANDLER_CONFIG.isThrowOnWriteLimitReached();
- }
-
- @GET
- @Produces("text/plain")
- public String getMessage() {
- checkIsOperating();
- return GREETING;
- }
-
- @POST
- @Consumes("multipart/form-data")
- @Produces("text/plain")
- @Path("form")
- public StreamingOutput getTextFromMultipart(Attachment att, @Context HttpHeaders httpHeaders, @Context final UriInfo info) {
- return produceText(att.getObject(InputStream.class), new Metadata(), preparePostHeaderMap(att, httpHeaders), info);
- }
-
- //this is equivalent to text-main in tika-app
- @PUT
- @Consumes("*/*")
- @Produces("text/plain")
- @Path("main")
- public StreamingOutput getTextMain(final InputStream is, @Context HttpHeaders httpHeaders, @Context final UriInfo info) {
- return produceTextMain(is, httpHeaders.getRequestHeaders(), info);
- }
-
- //this is equivalent to text-main (Boilerpipe handler) in tika-app
- @POST
- @Consumes("multipart/form-data")
- @Produces("text/plain")
- @Path("form/main")
- public StreamingOutput getTextMainFromMultipart(final Attachment att, @Context HttpHeaders httpHeaders, @Context final UriInfo info) {
- return produceTextMain(att.getObject(InputStream.class), preparePostHeaderMap(att, httpHeaders), info);
- }
-
- public StreamingOutput produceTextMain(final InputStream is, MultivaluedMap<String, String> httpHeaders, final UriInfo info) {
- final Parser parser = createParser();
- final Metadata metadata = new Metadata();
- final ParseContext context = new ParseContext();
-
- fillMetadata(parser, metadata, httpHeaders);
- fillParseContext(httpHeaders, metadata, context);
-
- logRequest(LOG, "/tika", metadata);
-
- return outputStream -> {
- Writer writer = new OutputStreamWriter(outputStream, UTF_8);
-
- ContentHandler handler = new BoilerpipeContentHandler(writer);
-
- parse(parser, LOG, info.getPath(), is, handler, metadata, context);
- };
- }
-
- @PUT
- @Consumes("*/*")
- @Produces("text/plain")
- public StreamingOutput getText(final InputStream is, @Context HttpHeaders httpHeaders, @Context final UriInfo info) {
- final Metadata metadata = new Metadata();
- return produceText(getInputStream(is, metadata, httpHeaders, info), metadata, httpHeaders.getRequestHeaders(), info);
- }
-
- public StreamingOutput produceText(final InputStream is, final Metadata metadata, MultivaluedMap<String, String> httpHeaders, final UriInfo info) {
- final Parser parser = createParser();
- final ParseContext context = new ParseContext();
-
- fillMetadata(parser, metadata, httpHeaders);
- fillParseContext(httpHeaders, metadata, context);
-
- logRequest(LOG, "/tika", metadata);
-
- return outputStream -> {
- Writer writer = new OutputStreamWriter(outputStream, UTF_8);
-
- BodyContentHandler body = new BodyContentHandler(new RichTextContentHandler(writer));
-
- parse(parser, LOG, info.getPath(), is, body, metadata, context);
- };
- }
-
- @POST
- @Consumes("multipart/form-data")
- @Produces("text/html")
- @Path("form")
- public StreamingOutput getHTMLFromMultipart(Attachment att, @Context HttpHeaders httpHeaders, @Context final UriInfo info) {
- return produceOutput(att.getObject(InputStream.class), new Metadata(), preparePostHeaderMap(att, httpHeaders), info, "html");
- }
-
- @PUT
- @Consumes("*/*")
- @Produces("text/html")
- public StreamingOutput getHTML(final InputStream is, @Context HttpHeaders httpHeaders, @Context final UriInfo info) {
- Metadata metadata = new Metadata();
- return produceOutput(getInputStream(is, metadata, httpHeaders, info), metadata, httpHeaders.getRequestHeaders(), info, "html");
- }
-
- @POST
- @Consumes("multipart/form-data")
- @Produces("text/xml")
- @Path("form")
- public StreamingOutput getXMLFromMultipart(Attachment att, @Context HttpHeaders httpHeaders, @Context final UriInfo info) {
- return produceOutput(att.getObject(InputStream.class), new Metadata(), preparePostHeaderMap(att, httpHeaders), info, "xml");
- }
-
- @PUT
- @Consumes("*/*")
- @Produces("text/xml")
- public StreamingOutput getXML(final InputStream is, @Context HttpHeaders httpHeaders, @Context final UriInfo info) {
- Metadata metadata = new Metadata();
- return produceOutput(getInputStream(is, metadata, httpHeaders, info), metadata, httpHeaders.getRequestHeaders(), info, "xml");
- }
-
- @POST
- @Consumes("multipart/form-data")
- @Produces("application/json")
- @Path("form{" + HANDLER_TYPE_PARAM + " : (\\w+)?}")
- public Metadata getJsonFromMultipart(Attachment att, @Context HttpHeaders httpHeaders, @Context final UriInfo info, @PathParam(HANDLER_TYPE_PARAM) String handlerTypeName)
- throws IOException, TikaException {
- Metadata metadata = new Metadata();
- parseToMetadata(getInputStream(att.getObject(InputStream.class), metadata, httpHeaders, info), metadata, preparePostHeaderMap(att, httpHeaders), info, handlerTypeName);
- TikaResource
- .getConfig()
- .getMetadataFilter()
- .filter(metadata);
- return metadata;
- }
-
- @PUT
- @Consumes("*/*")
- @Produces("application/json")
- @Path("{" + HANDLER_TYPE_PARAM + " : (\\w+)?}")
- public Metadata getJson(final InputStream is, @Context HttpHeaders httpHeaders, @Context final UriInfo info, @PathParam(HANDLER_TYPE_PARAM) String handlerTypeName)
- throws IOException, TikaException {
- Metadata metadata = new Metadata();
- parseToMetadata(getInputStream(is, metadata, httpHeaders, info), metadata, httpHeaders.getRequestHeaders(), info, handlerTypeName);
- TikaResource
- .getConfig()
- .getMetadataFilter()
- .filter(metadata);
- return metadata;
- }
-
- private void parseToMetadata(InputStream inputStream, Metadata metadata, MultivaluedMap<String, String> httpHeaders, UriInfo info, String handlerTypeName) throws IOException {
- final Parser parser = createParser();
- final ParseContext context = new ParseContext();
-
- fillMetadata(parser, metadata, httpHeaders);
- fillParseContext(httpHeaders, metadata, context);
-
- logRequest(LOG, "/tika", metadata);
- int writeLimit = -1;
- boolean throwOnWriteLimitReached = getThrowOnWriteLimitReached(httpHeaders);
- if (httpHeaders.containsKey("writeLimit")) {
- writeLimit = Integer.parseInt(httpHeaders.getFirst("writeLimit"));
- }
-
- BasicContentHandlerFactory.HANDLER_TYPE type = BasicContentHandlerFactory.parseHandlerType(handlerTypeName, DEFAULT_HANDLER_TYPE);
- BasicContentHandlerFactory fact = new BasicContentHandlerFactory(type, writeLimit, throwOnWriteLimitReached, context);
- ContentHandler contentHandler = fact.getNewContentHandler();
-
- try {
- parse(parser, LOG, info.getPath(), inputStream, contentHandler, metadata, context);
- } catch (TikaServerParseException e) {
- Throwable cause = e.getCause();
- boolean writeLimitReached = false;
- if (WriteLimitReachedException.isWriteLimitReached(cause)) {
- metadata.set(TikaCoreProperties.WRITE_LIMIT_REACHED, "true");
- writeLimitReached = true;
- }
- if (TIKA_SERVER_CONFIG.isReturnStackTrace()) {
- if (cause != null) {
- metadata.add(TikaCoreProperties.CONTAINER_EXCEPTION, ExceptionUtils.getStackTrace(cause));
- } else {
- metadata.add(TikaCoreProperties.CONTAINER_EXCEPTION, ExceptionUtils.getStackTrace(e));
- }
- } else if (!writeLimitReached) {
- throw e;
- }
- } catch (OutOfMemoryError e) {
- if (TIKA_SERVER_CONFIG.isReturnStackTrace()) {
- metadata.add(TikaCoreProperties.CONTAINER_EXCEPTION, ExceptionUtils.getStackTrace(e));
- } else {
- throw e;
- }
- } finally {
- metadata.add(TikaCoreProperties.TIKA_CONTENT, contentHandler.toString());
- }
- }
-
- private StreamingOutput produceOutput(final InputStream is, Metadata metadata, final MultivaluedMap<String, String> httpHeaders, final UriInfo info, final String format) {
- final Parser parser = createParser();
- final ParseContext context = new ParseContext();
-
- fillMetadata(parser, metadata, httpHeaders);
- fillParseContext(httpHeaders, metadata, context);
-
-
- logRequest(LOG, "/tika", metadata);
-
- return outputStream -> {
- Writer writer = new OutputStreamWriter(outputStream, UTF_8);
- ContentHandler content;
-
- try {
- SAXTransformerFactory factory = XMLReaderUtils.getSAXTransformerFactory();
- TransformerHandler handler = factory.newTransformerHandler();
- handler
- .getTransformer()
- .setOutputProperty(OutputKeys.METHOD, format);
- handler
- .getTransformer()
- .setOutputProperty(OutputKeys.INDENT, "yes");
- handler
- .getTransformer()
- .setOutputProperty(OutputKeys.ENCODING, UTF_8.name());
- handler
- .getTransformer()
- .setOutputProperty(OutputKeys.VERSION, "1.1");
- handler.setResult(new StreamResult(writer));
- content = new ExpandedTitleContentHandler(handler);
- } catch (TransformerConfigurationException | TikaException e) {
- throw new WebApplicationException(e);
- }
-
- parse(parser, LOG, info.getPath(), is, content, metadata, context);
- };
- }
-
- /**
- * Prepares a multivalued map, combining attachment headers and request headers.
- * Gives priority to attachment headers.
- *
- * @param att the attachment.
- * @param httpHeaders the http headers, fetched from context.
- * @return the case insensitive MetadataMap containing combined headers.
- */
- private MetadataMap<String, String> preparePostHeaderMap(Attachment att, HttpHeaders httpHeaders) {
- if (att == null && httpHeaders == null) {
- return null;
- }
- MetadataMap<String, String> finalHeaders = new MetadataMap<>(false, true);
- if (httpHeaders != null && httpHeaders.getRequestHeaders() != null) {
- finalHeaders.putAll(httpHeaders.getRequestHeaders());
- }
- if (att != null && att.getHeaders() != null) {
- finalHeaders.putAll(att.getHeaders());
- }
- return finalHeaders;
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaServerResource.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaServerResource.java
deleted file mode 100644
index 537b36e41..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaServerResource.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core.resource;
-
-/**
- * Stub interface to allow for loading of resources via SPI
- */
-public interface TikaServerResource {
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaServerStatus.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaServerStatus.java
deleted file mode 100644
index dd8ac68f3..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaServerStatus.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core.resource;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-import jakarta.ws.rs.GET;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.Produces;
-
-import org.apache.tika.server.core.ServerStatus;
-
-@Path("/status")
-public class TikaServerStatus {
- private final ServerStatus serverStatus;
-
- public TikaServerStatus(ServerStatus serverStatus) {
- this.serverStatus = serverStatus;
- }
-
- @GET
- @Produces("application/json")
- public Map<String, Object> getStatus() {
- Map<String, Object> map = new LinkedHashMap<>();
- map.put("server_id", serverStatus.getServerId());
- map.put("status", serverStatus.getStatus());
- map.put("millis_since_last_parse_started", serverStatus.getMillisSinceLastParseStarted());
- map.put("files_processed", serverStatus.getFilesProcessed());
- map.put("num_restarts", serverStatus.getNumRestarts());
- return map;
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaVersion.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaVersion.java
deleted file mode 100644
index c894c2a88..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaVersion.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core.resource;
-
-import jakarta.ws.rs.GET;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.Produces;
-
-import org.apache.tika.Tika;
-
-@Path("/version")
-public class TikaVersion {
- private Tika tika;
-
- public TikaVersion() {
- this.tika = new Tika(TikaResource.getConfig());
- }
-
- @GET
- @Produces("text/plain")
- public String getVersion() {
- TikaResource.checkIsOperating();
- return tika.toString();
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaWelcome.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaWelcome.java
deleted file mode 100644
index 207629062..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaWelcome.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core.resource;
-
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import jakarta.ws.rs.DELETE;
-import jakarta.ws.rs.GET;
-import jakarta.ws.rs.HEAD;
-import jakarta.ws.rs.OPTIONS;
-import jakarta.ws.rs.POST;
-import jakarta.ws.rs.PUT;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.Produces;
-import org.apache.cxf.jaxrs.lifecycle.ResourceProvider;
-
-import org.apache.tika.Tika;
-import org.apache.tika.server.core.HTMLHelper;
-
-/**
- * <p>Provides a basic welcome to the Apache Tika Server.</p>
- */
-@Path("/")
-public class TikaWelcome {
- private static final String DOCS_URL = "https://wiki.apache.org/tika/TikaJAXRS";
-
- private static final Map<Class<? extends Annotation>, String> HTTP_METHODS = new HashMap<>();
-
- static {
- HTTP_METHODS.put(DELETE.class, "DELETE");
- HTTP_METHODS.put(GET.class, "GET");
- HTTP_METHODS.put(HEAD.class, "HEAD");
- HTTP_METHODS.put(OPTIONS.class, "OPTIONS");
- HTTP_METHODS.put(POST.class, "POST");
- HTTP_METHODS.put(PUT.class, "PUT");
- }
-
- private Tika tika;
- private HTMLHelper html;
- private List<Class<?>> endpoints = new LinkedList<>();
-
- public TikaWelcome(List<ResourceProvider> rCoreProviders) {
- this.tika = new Tika(TikaResource.getConfig());
- this.html = new HTMLHelper();
- for (ResourceProvider rp : rCoreProviders) {
- this.endpoints.add(rp.getResourceClass());
- }
- }
-
- protected List<Endpoint> identifyEndpoints() {
- List<Endpoint> found = new ArrayList<>();
- for (Class<?> endpoint : endpoints) {
- Path p = endpoint.getAnnotation(Path.class);
- String basePath = null;
- if (p != null) {
- basePath = p
- .value()
- .endsWith("/") ? p
- .value()
- .substring(0, p
- .value()
- .length() - 2) : p.value();
- }
-
- for (Method m : endpoint.getMethods()) {
- String httpMethod = null;
- String methodPath = null;
- String[] produces = null;
-
- for (Annotation a : m.getAnnotations()) {
- for (Class<? extends Annotation> httpMethAnn : HTTP_METHODS.keySet()) {
- if (httpMethAnn.isInstance(a)) {
- httpMethod = HTTP_METHODS.get(httpMethAnn);
- }
- }
- if (a instanceof Path) {
- methodPath = ((Path) a).value();
- }
- if (a instanceof Produces) {
- produces = ((Produces) a).value();
- }
- }
-
- if (httpMethod != null) {
- String mPath = basePath;
- if (mPath == null) {
- mPath = "";
- }
- if (methodPath != null) {
- if (methodPath.startsWith("/")) {
- mPath += methodPath;
- } else {
- mPath += "/" + methodPath;
- }
- }
- if (produces == null) {
- produces = new String[0];
- }
- found.add(new Endpoint(endpoint, m, mPath, httpMethod, produces));
- }
- }
- }
- found.sort(Comparator
- .comparing((Endpoint e) -> e.path)
- .thenComparing(e -> e.methodName));
- return found;
- }
-
- @GET
- @Produces("text/html")
- public String getWelcomeHTML() {
- TikaResource.checkIsOperating();
-
- StringBuffer h = new StringBuffer();
- String tikaVersion = tika.toString();
-
- html.generateHeader(h, "Welcome to the " + tikaVersion + " Server");
-
- h.append("<p>For endpoints, please see <a href=\"");
- h.append(DOCS_URL);
- h.append("\">");
- h.append(DOCS_URL);
- h.append("</a>");
-
- // TIKA-1269 -- Miredot documentation
- // As the SNAPSHOT endpoints are updated, please update the website by running
- // the server tests and doing step 12.6 of https://wiki.apache.org/tika/ReleaseProcess.
- Pattern p = Pattern.compile("\\d+\\.\\d+");
- Matcher m = p.matcher(tikaVersion);
- if (m.find()) {
- String versionNumber = m.group();
- String miredot = "https://tika.apache.org/" + versionNumber + "/miredot/index.html";
- h
- .append(" and <a href=\"")
- .append(miredot)
- .append("\">")
- .append(miredot)
- .append("</a>");
- }
- h.append("</p>\n");
-
- h.append("<ul>\n");
- for (Endpoint e : identifyEndpoints()) {
- h.append("<li><b>");
- h.append(e.httpMethod);
- h.append("</b> <i><a href=\"");
- h.append(e.path);
- h.append("\">");
- h.append(e.path);
- h.append("</a></i><br />");
- h.append("Class: ");
- h.append(e.className);
- h.append("<br />Method: ");
- h.append(e.methodName);
- for (String produces : e.produces) {
- h.append("<br />Produces: ");
- h.append(produces);
- }
- h.append("</li>\n");
- }
- h.append("</ul>\n");
-
- html.generateFooter(h);
- return h.toString();
- }
-
- @GET
- @Produces("text/plain")
- public String getWelcomePlain() {
- TikaResource.checkIsOperating();
- StringBuilder text = new StringBuilder();
-
- text.append(tika.toString());
- text.append("\n");
- text.append("For endpoints, please see ");
- text.append(DOCS_URL);
- text.append("\n\n");
-
- for (Endpoint e : identifyEndpoints()) {
- text.append(e.httpMethod);
- text.append(" ");
- text.append(e.path);
- text.append("\n");
- for (String produces : e.produces) {
- text.append(" => ");
- text.append(produces);
- text.append("\n");
- }
- }
-
- return text.toString();
- }
-
- protected static class Endpoint {
- public final String className;
- public final String methodName;
- public final String path;
- public final String httpMethod;
- public final List<String> produces;
-
- protected Endpoint(Class<?> endpoint, Method method, String path, String httpMethod, String[] produces) {
- this.className = endpoint.getCanonicalName();
- this.methodName = method.getName();
- this.path = path;
- this.httpMethod = httpMethod;
- this.produces = Collections.unmodifiableList(Arrays.asList(produces));
- }
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TranslateResource.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TranslateResource.java
deleted file mode 100644
index d19ead2f7..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TranslateResource.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core.resource;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-
-import jakarta.ws.rs.Consumes;
-import jakarta.ws.rs.POST;
-import jakarta.ws.rs.PUT;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.PathParam;
-import jakarta.ws.rs.Produces;
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.LoadErrorHandler;
-import org.apache.tika.config.ServiceLoader;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.langdetect.optimaize.OptimaizeLangDetector;
-import org.apache.tika.language.detect.LanguageResult;
-import org.apache.tika.language.translate.Translator;
-import org.apache.tika.server.core.ServerStatus;
-
-@Path("/translate")
-public class TranslateResource {
- private static final Logger LOG = LoggerFactory.getLogger(TranslateResource.class);
- private final ServerStatus serverStatus;
- private final long timeoutMillis;
- private Translator defaultTranslator;
- private ServiceLoader loader;
-
- public TranslateResource(ServerStatus serverStatus, long timeoutMillis) {
- this.loader = new ServiceLoader(ServiceLoader.class.getClassLoader(), LoadErrorHandler.WARN);
- this.defaultTranslator = TikaResource
- .getConfig()
- .getTranslator();
- this.serverStatus = serverStatus;
- this.timeoutMillis = timeoutMillis;
- }
-
- @PUT
- @POST
- @Path("/all/{translator}/{src}/{dest}")
- @Consumes("*/*")
- @Produces("text/plain")
- public String translate(final InputStream is, @PathParam("translator") String translator, @PathParam("src") String sLang, @PathParam("dest") String dLang)
- throws TikaException, IOException {
- return doTranslate(IOUtils.toString(is, UTF_8), translator, sLang, dLang);
-
- }
-
- @PUT
- @POST
- @Path("/all/{translator}/{dest}")
- @Consumes("*/*")
- @Produces("text/plain")
- public String autoTranslate(final InputStream is, @PathParam("translator") String translator, @PathParam("dest") String dLang) throws TikaException, IOException {
- final String content = IOUtils.toString(is, UTF_8);
- LanguageResult language = new OptimaizeLangDetector()
- .loadModels()
- .detect(content);
- if (language.isUnknown()) {
- throw new TikaException("Unable to detect language to use for translation of text");
- }
-
- String sLang = language.getLanguage();
- LOG.info("LanguageIdentifier: detected source lang: [{}]", sLang);
-
- return doTranslate(content, translator, sLang, dLang);
- }
-
- private String doTranslate(String content, String translator, String sLang, String dLang) throws TikaException, IOException {
- LOG.info("Using translator: [{}]: src: [{}]: dest: [{}]", translator, sLang, dLang);
- Translator translate = byClassName(translator);
- if (translate == null) {
- translate = this.defaultTranslator;
- LOG.info("Using default translator");
- }
- TikaResource.checkIsOperating();
- long taskId = serverStatus.start(ServerStatus.TASK.TRANSLATE, null, timeoutMillis);
- try {
- return translate.translate(content, sLang, dLang);
- } catch (OutOfMemoryError e) {
- serverStatus.setStatus(ServerStatus.STATUS.ERROR);
- throw e;
- } finally {
- serverStatus.complete(taskId);
- }
- }
-
- private Translator byClassName(String className) {
- List<Translator> translators = loader.loadStaticServiceProviders(Translator.class);
- for (Translator t : translators) {
- if (t
- .getClass()
- .getName()
- .equals(className)) {
- return t;
- }
- }
- return null;
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java
deleted file mode 100644
index a2e3064d6..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core.resource;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.tika.server.core.resource.TikaResource.fillMetadata;
-import static org.apache.tika.server.core.resource.TikaResource.fillParseContext;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-
-import jakarta.ws.rs.PUT;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.Produces;
-import jakarta.ws.rs.WebApplicationException;
-import jakarta.ws.rs.core.Context;
-import jakarta.ws.rs.core.HttpHeaders;
-import jakarta.ws.rs.core.Response;
-import jakarta.ws.rs.core.UriInfo;
-import org.apache.commons.csv.CSVFormat;
-import org.apache.commons.csv.CSVPrinter;
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream;
-import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
-
-import org.apache.tika.exception.TikaMemoryLimitException;
-import org.apache.tika.extractor.DefaultEmbeddedStreamTranslator;
-import org.apache.tika.extractor.EmbeddedDocumentExtractor;
-import org.apache.tika.extractor.EmbeddedStreamTranslator;
-import org.apache.tika.io.BoundedInputStream;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.mime.MimeTypeException;
-import org.apache.tika.parser.DigestingParser;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.apache.tika.sax.BodyContentHandler;
-import org.apache.tika.sax.RichTextContentHandler;
-
-@Path("/unpack")
-public class UnpackerResource {
- public static final String TEXT_FILENAME = "__TEXT__";
- public static final String META_FILENAME = "__METADATA__";
-
- public static final String UNPACK_MAX_BYTES_KEY = "unpackMaxBytes";
-
- private static final long DEFAULT_MAX_ATTACHMENT_BYTES = 100 * 1024 * 1024;
-
- private static final Logger LOG = LoggerFactory.getLogger(UnpackerResource.class);
-
- public static void metadataToCsv(Metadata metadata, OutputStream outputStream) throws IOException {
- CSVPrinter writer = new CSVPrinter(new OutputStreamWriter(outputStream, UTF_8), CSVFormat.EXCEL);
-
- for (String name : metadata.names()) {
- String[] values = metadata.getValues(name);
- ArrayList<String> list = new ArrayList<>(values.length + 1);
- list.add(name);
- list.addAll(Arrays.asList(values));
- writer.printRecord(list);
- }
-
- writer.close();
- }
-
- @Path("/{id:(/.*)?}")
- @PUT
- @Produces({"application/zip", "application/x-tar"})
- public Map<String, byte[]> unpack(InputStream is, @Context HttpHeaders httpHeaders, @Context UriInfo info) throws Exception {
- return process(TikaResource.getInputStream(is, new Metadata(), httpHeaders, info), httpHeaders, info, false);
- }
-
- @Path("/all{id:(/.*)?}")
- @PUT
- @Produces({"application/zip", "application/x-tar"})
- public Map<String, byte[]> unpackAll(InputStream is, @Context HttpHeaders httpHeaders, @Context UriInfo info) throws Exception {
- return process(TikaResource.getInputStream(is, new Metadata(), httpHeaders, info), httpHeaders, info, true);
- }
-
- private Map<String, byte[]> process(InputStream is, @Context HttpHeaders httpHeaders, @Context UriInfo info, boolean saveAll) throws Exception {
- Metadata metadata = new Metadata();
- ParseContext pc = new ParseContext();
- long unpackMaxBytes = DEFAULT_MAX_ATTACHMENT_BYTES;
- String unpackMaxBytesString = httpHeaders
- .getRequestHeaders()
- .getFirst(UNPACK_MAX_BYTES_KEY);
- if (!StringUtils.isBlank(unpackMaxBytesString)) {
- unpackMaxBytes = Long.parseLong(unpackMaxBytesString);
- if (unpackMaxBytes > Integer.MAX_VALUE) {
- throw new IllegalArgumentException("Can't request value > than Integer" + ".MAX_VALUE : " + unpackMaxBytes);
- } else if (unpackMaxBytes < 0) {
- throw new IllegalArgumentException("Can't request value < 0: " + unpackMaxBytes);
- }
- }
- Parser parser = TikaResource.createParser();
- if (parser instanceof DigestingParser) {
- //no need to digest for unwrapping
- parser = ((DigestingParser) parser).getWrappedParser();
- }
- fillMetadata(parser, metadata, httpHeaders.getRequestHeaders());
- fillParseContext(httpHeaders.getRequestHeaders(), metadata, pc);
-
- TikaResource.logRequest(LOG, "/unpack", metadata);
- //even though we aren't currently parsing embedded documents,
- //we need to add this to allow for "inline" use of other parsers.
- pc.set(Parser.class, parser);
- ContentHandler ch;
- UnsynchronizedByteArrayOutputStream text = UnsynchronizedByteArrayOutputStream
- .builder()
- .get();
-
- if (saveAll) {
- ch = new BodyContentHandler(new RichTextContentHandler(new OutputStreamWriter(text, UTF_8)));
- } else {
- ch = new DefaultHandler();
- }
-
- Map<String, byte[]> files = new HashMap<>();
- MutableInt count = new MutableInt();
-
- pc.set(EmbeddedDocumentExtractor.class, new MyEmbeddedDocumentExtractor(count, files, unpackMaxBytes));
-
- TikaResource.parse(parser, LOG, info.getPath(), is, ch, metadata, pc);
-
- if (count.intValue() == 0 && !saveAll) {
- throw new WebApplicationException(Response.Status.NO_CONTENT);
- }
-
- if (saveAll) {
- files.put(TEXT_FILENAME, text.toByteArray());
-
- UnsynchronizedByteArrayOutputStream metaStream = UnsynchronizedByteArrayOutputStream
- .builder()
- .get();
- metadataToCsv(metadata, metaStream);
-
- files.put(META_FILENAME, metaStream.toByteArray());
- }
-
- return files;
- }
-
- private static class MyEmbeddedDocumentExtractor implements EmbeddedDocumentExtractor {
- private final MutableInt count;
- private final Map<String, byte[]> zout;
-
- private final long unpackMaxBytes;
- private final EmbeddedStreamTranslator embeddedStreamTranslator = new DefaultEmbeddedStreamTranslator();
-
- MyEmbeddedDocumentExtractor(MutableInt count, Map<String, byte[]> zout, long unpackMaxBytes) {
- this.count = count;
- this.zout = zout;
- this.unpackMaxBytes = unpackMaxBytes;
- }
-
- public boolean shouldParseEmbedded(Metadata metadata) {
- return true;
- }
-
- @Override
- public void parseEmbedded(TikaInputStream tis, ContentHandler contentHandler, Metadata metadata, boolean b) throws SAXException, IOException {
- UnsynchronizedByteArrayOutputStream bos = UnsynchronizedByteArrayOutputStream
- .builder()
- .get();
-
- BoundedInputStream bis = new BoundedInputStream(unpackMaxBytes, tis);
- IOUtils.copy(bis, bos);
- if (bis.hasHitBound()) {
- throw new IOException(new TikaMemoryLimitException(
- "An attachment is longer than " + "'unpackMaxBytes' (default=100MB, actual=" + unpackMaxBytes + "). " + "If you need to increase this " +
- "limit, add a header to your request, such as: unpackMaxBytes: " + "1073741824. There is a hard limit of 2GB."));
- }
- byte[] data = bos.toByteArray();
-
- String name = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
- String contentType = metadata.get(org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE);
-
- if (name == null) {
- name = Integer.toString(count.intValue());
- }
-
- if (!name.contains(".") && contentType != null) {
- try {
- String ext = TikaResource
- .getConfig()
- .getMimeRepository()
- .forName(contentType)
- .getExtension();
-
- if (ext != null) {
- name += ext;
- }
- } catch (MimeTypeException e) {
- LOG.warn("Unexpected MimeTypeException", e);
- }
- }
- try (TikaInputStream is = TikaInputStream.get(data)) {
- if (embeddedStreamTranslator.shouldTranslate(is, metadata)) {
- InputStream translated = embeddedStreamTranslator.translate(UnsynchronizedByteArrayInputStream.builder().setByteArray(data).get(), metadata);
- UnsynchronizedByteArrayOutputStream bos2 = UnsynchronizedByteArrayOutputStream
- .builder()
- .get();
- IOUtils.copy(translated, bos2);
- data = bos2.toByteArray();
- }
- }
-
- final String finalName = getFinalName(name, zout);
-
- if (data.length > 0) {
- zout.put(finalName, data);
- count.increment();
- }
- }
-
- private String getFinalName(String name, Map<String, byte[]> zout) {
- name = name.replaceAll("\u0000", " ");
- String normalizedName = FilenameUtils.normalize(name);
-
- if (normalizedName == null) {
- normalizedName = FilenameUtils.getName(name);
- }
-
- if (normalizedName == null) {
- normalizedName = count.toString();
- }
- //strip off initial C:/ or ~/ or /
- int prefixLength = FilenameUtils.getPrefixLength(normalizedName);
- if (prefixLength > -1) {
- normalizedName = normalizedName.substring(prefixLength);
- }
- if (zout.containsKey(normalizedName)) {
- return UUID
- .randomUUID()
- .toString() + "-" + normalizedName;
- }
- return normalizedName;
- }
-
-/* protected void copy(DirectoryEntry sourceDir, DirectoryEntry destDir)
- throws IOException {
- for (Entry entry : sourceDir) {
- if (entry instanceof DirectoryEntry) {
- // Need to recurse
- DirectoryEntry newDir = destDir.createDirectory(entry.getName());
- copy((DirectoryEntry) entry, newDir);
- } else {
- // Copy entry
- try (InputStream contents = new DocumentInputStream((DocumentEntry) entry)) {
- destDir.createDocument(entry.getName(), contents);
- }
- }
- }
- }*/
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/writer/CSVMessageBodyWriter.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/writer/CSVMessageBodyWriter.java
deleted file mode 100644
index ba9ce6727..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/writer/CSVMessageBodyWriter.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core.writer;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import jakarta.ws.rs.Produces;
-import jakarta.ws.rs.WebApplicationException;
-import jakarta.ws.rs.core.MediaType;
-import jakarta.ws.rs.core.MultivaluedMap;
-import jakarta.ws.rs.ext.MessageBodyWriter;
-import jakarta.ws.rs.ext.Provider;
-import org.apache.commons.csv.CSVFormat;
-import org.apache.commons.csv.CSVPrinter;
-
-import org.apache.tika.metadata.Metadata;
-
-@Provider
-@Produces("text/csv")
-public class CSVMessageBodyWriter implements MessageBodyWriter<Metadata> {
-
- public boolean isWriteable(Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {
- return Metadata.class.isAssignableFrom(type);
- }
-
- public long getSize(Metadata data, Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {
- return -1;
- }
-
- @Override
- @SuppressWarnings("resource")
- public void writeTo(Metadata metadata, Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, Object> httpHeaders,
- OutputStream entityStream) throws IOException, WebApplicationException {
-
- CSVPrinter writer = new CSVPrinter(new OutputStreamWriter(entityStream, UTF_8), CSVFormat.EXCEL);
-
- for (String name : metadata.names()) {
- String[] values = metadata.getValues(name);
- ArrayList<String> list = new ArrayList<>(values.length + 1);
- list.add(name);
- list.addAll(Arrays.asList(values));
- writer.printRecord(list);
- }
-
- // Don't close, just flush the stream
- writer.flush();
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONMessageBodyWriter.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONMessageBodyWriter.java
deleted file mode 100644
index 942cea29b..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONMessageBodyWriter.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core.writer;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Type;
-
-import jakarta.ws.rs.Produces;
-import jakarta.ws.rs.WebApplicationException;
-import jakarta.ws.rs.core.MediaType;
-import jakarta.ws.rs.core.MultivaluedMap;
-import jakarta.ws.rs.ext.MessageBodyWriter;
-import jakarta.ws.rs.ext.Provider;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.serialization.JsonMetadata;
-
-@Provider
-@Produces(MediaType.APPLICATION_JSON)
-public class JSONMessageBodyWriter implements MessageBodyWriter<Metadata> {
-
- public boolean isWriteable(Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {
- return Metadata.class.isAssignableFrom(type);
- }
-
- public long getSize(Metadata data, Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {
- return -1;
- }
-
- @Override
- public void writeTo(Metadata metadata, Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, Object> httpHeaders,
- OutputStream entityStream) throws IOException, WebApplicationException {
- try (Writer writer = new OutputStreamWriter(entityStream, UTF_8)) {
- JsonMetadata.toJson(metadata, writer);
- }
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONObjWriter.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONObjWriter.java
deleted file mode 100644
index 8ed053678..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONObjWriter.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core.writer;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Type;
-import java.util.Map;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import jakarta.ws.rs.Produces;
-import jakarta.ws.rs.WebApplicationException;
-import jakarta.ws.rs.core.MediaType;
-import jakarta.ws.rs.core.MultivaluedMap;
-import jakarta.ws.rs.ext.MessageBodyWriter;
-import jakarta.ws.rs.ext.Provider;
-
-import org.apache.tika.metadata.Metadata;
-
-@Provider
-@Produces(MediaType.APPLICATION_JSON)
-public class JSONObjWriter implements MessageBodyWriter<Map<String, Object>> {
-
-
- public boolean isWriteable(Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {
- return Map.class.isAssignableFrom(type);
- }
-
- public long getSize(Metadata data, Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {
- return -1;
- }
-
- @Override
- public void writeTo(Map<String, Object> map, Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, Object> httpHeaders,
- OutputStream entityStream) throws IOException, WebApplicationException {
- try (Writer writer = new OutputStreamWriter(entityStream, UTF_8)) {
- ObjectMapper objectMapper = new ObjectMapper();
- objectMapper
- .writerWithDefaultPrettyPrinter()
- .writeValue(writer, map);
- }
- }
-}
diff --git a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/writer/MetadataListMessageBodyWriter.java b/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/writer/MetadataListMessageBodyWriter.java
deleted file mode 100644
index 1e0a139db..000000000
--- a/tika-server-old/tika-server-core/src/main/java/org/apache/tika/server/core/writer/MetadataListMessageBodyWriter.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.server.core.writer;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.lang.annotation.Annotation;
... 17023 lines suppressed ...
(tika) 04/04: TIKA-4334: move grpc out of pipes and into tika-server, implement more controller
Posted by nd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ndipiazza pushed a commit to branch TIKA-4334
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 7c1efa31f344b737be4a7f602b3a7941357e5a49
Author: Nicholas DiPiazza <nd...@apache.org>
AuthorDate: Wed Oct 1 07:53:58 2025 -0500
TIKA-4334: move grpc out of pipes and into tika-server, implement more controller
---
tika-pipes/pom.xml | 1 -
tika-server/README.md | 89 ---------
tika-server/pom.xml | 1 +
.../tika-server-grpc}/README.md | 0
.../tika-server-grpc}/corpa-files/014760.docx | Bin
.../tika-server-grpc}/corpa-files/017091.docx | Bin
.../tika-server-grpc}/corpa-files/017097.docx | Bin
.../tika-server-grpc}/corpa-files/018367.docx | Bin
.../tika-server-grpc}/corpa-files/018370.docx | Bin
.../tika-server-grpc}/corpa-files/018371.docx | Bin
.../tika-server-grpc}/corpa-files/021524.docx | Bin
.../corpa-files/file-that-confuses-tika.log | 0
.../tika-server-grpc}/docker-build/Dockerfile | 0
.../tika-server-grpc}/docker-build/docker-build.sh | 0
.../docker-build/start-tika-grpc.sh | 0
.../tika-server-grpc}/pom.xml | 4 +-
.../apache/tika/pipes/TikaPipesApplication.java | 0
.../tika/pipes/grpc/TikaGrpcServerCustomizer.java | 0
.../apache/tika/pipes/grpc/TikaGrpcService.java | 0
.../java/org/apache/tika/pipes/job/JobStatus.java | 0
.../tika/pipes/model/FetchAndParseStatus.java | 0
.../parser/TikaPipesApplicationConfiguration.java | 0
.../plugin/ClasspathPluginPropertiesFinder.java | 0
.../tika/pipes/plugin/GrpcPluginManager.java | 7 +-
.../org/apache/tika/pipes/plugin/PluginConfig.java | 0
.../apache/tika/pipes/repo/EmitterRepository.java | 0
.../apache/tika/pipes/repo/FetcherRepository.java | 0
.../pipes/repo/IgniteRepositoryConfiguration.java | 0
.../tika/pipes/repo/JobStatusRepository.java | 0
.../tika/pipes/repo/PipeIteratorRepository.java | 0
.../src/main/resources/application.yaml | 0
.../tika/pipes/TikaPipesIntegrationTestBase.java | 0
.../pipes/grpc/TikaGrpcServerEmittersCrudTest.java | 0
.../grpc/TikaGrpcServerFetchAndParseTest.java | 0
.../pipes/grpc/TikaGrpcServerFetchersCrudTest.java | 0
.../pipes/grpc/TikaGrpcServerJsonSchemaTest.java | 0
.../grpc/TikaGrpcServerPipeIteratorsCrudTest.java | 0
.../tika/pipes/grpc/TikaGrpcServerPipeJobTest.java | 0
tika-server/tika-server-spring/pom.xml | 21 +-
.../controller/InformationServicesController.java | 73 -------
.../controller/LanguageResourceController.java | 37 +++-
.../tika/server/controller/MetadataController.java | 222 ++++++++++++++++++++-
.../exception/TikaServerRuntimeException.java | 35 ++++
.../apache/tika/server/service/ParserService.java | 48 +++++
.../server/service/TikaMetadataFillerService.java | 124 ++++++++++++
...formationServicesControllerIntegrationTest.java | 39 ----
.../LanguageResourceControllerIntegrationTest.java | 105 +++++++++-
.../MetadataControllerIntegrationTest.java | 221 +++++++++++++++++++-
.../src/test/resources/test-documents/english.txt | 2 +
.../src/test/resources/test-documents/french.txt | 2 +
.../test-documents/password-protected.doc | Bin 0 -> 22016 bytes
.../src/test/resources/test-documents/test.doc | Bin 0 -> 9216 bytes
52 files changed, 798 insertions(+), 233 deletions(-)
diff --git a/tika-pipes/pom.xml b/tika-pipes/pom.xml
index feb6c2537..f8dcc35ea 100644
--- a/tika-pipes/pom.xml
+++ b/tika-pipes/pom.xml
@@ -69,7 +69,6 @@
<module>tika-pipes-fetchers</module>
<module>tika-pipes-emitters</module>
<module>tika-pipes-proto</module>
- <module>tika-pipes-grpc</module>
<module>tika-pipes-cli</module>
</modules>
</project>
diff --git a/tika-server/README.md b/tika-server/README.md
deleted file mode 100644
index 7ecb77312..000000000
--- a/tika-server/README.md
+++ /dev/null
@@ -1,89 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-# Apache Tika Server
-
-https://cwiki.apache.org/confluence/display/TIKA/TikaJAXRS
-
-Running
--------
-```
-$ java -jar tika-server/target/tika-server.jar --help
- usage: tikaserver
- -?,--help this help message
- -h,--host <arg> host name (default = localhost)
- -l,--log <arg> request URI log level ('debug' or 'info')
- -p,--port <arg> listen port (default = 9998)
- -s,--includeStack whether or not to return a stack trace
- if there is an exception during 'parse'
-```
-
-Running via Docker
-------------------
-Assuming you have Docker installed, you can use a prebuilt image:
-
-`docker run -d -p 9998:9998 apache/tika`
-
-This will load Apache Tika Server and expose its interface on:
-
-`http://localhost:9998`
-
-You may also be interested in the https://github.com/apache/tika-docker project
-which provides prebuilt Docker images.
-
-Installing as a Service on Linux
------------------------
-To run as a service on Linux you need to run the `install_tika_service.sh` script.
-
-Assuming you have the binary distribution like `tika-server-2.0.0-bin.tgz`,
-then you can extract the install script via:
-
-`tar xzf tika-server-2.0.0-bin.tgz --strip-components=2 tika-server-2.0.0-bin/bin/install_tika_service.sh`
-
-and then run the installation process via:
-
-`./install_tika_service.sh ./tika-server-2.0.0-bin.tgz`
-
-
-Usage
------
-Usage examples from command line with `curl` utility:
-
-* Extract plain text:
-`curl -T price.xls http://localhost:9998/tika`
-
-* Extract text with mime-type hint:
-`curl -v -H "Content-type: application/vnd.openxmlformats-officedocument.wordprocessingml.document" -T document.docx http://localhost:9998/tika`
-
-* Get all document attachments as ZIP-file:
-`curl -v -T Doc1_ole.doc http://localhost:9998/unpacker > /var/tmp/x.zip`
-
-* Extract metadata to CSV format:
-`curl -T price.xls http://localhost:9998/meta`
-
-* Detect media type from CSV format using file extension hint:
-`curl -X PUT -H "Content-Disposition: attachment; filename=foo.csv" --upload-file foo.csv http://localhost:9998/detect/stream`
-
-
-HTTP Return Codes
------------------
-`200` - Ok
-`204` - No content (for example when we are unpacking file without attachments)
-`415` - Unknown file type
-`422` - Unparsable document of known type (password protected documents and unsupported versions like Biff5 Excel)
-`500` - Internal error
diff --git a/tika-server/pom.xml b/tika-server/pom.xml
index 32593068f..68fcac963 100644
--- a/tika-server/pom.xml
+++ b/tika-server/pom.xml
@@ -38,6 +38,7 @@
<modelVersion>4.0.0</modelVersion>
<modules>
<module>tika-server-spring</module>
+ <module>tika-server-grpc</module>
</modules>
<parent>
diff --git a/tika-pipes/tika-pipes-grpc/README.md b/tika-server/tika-server-grpc/README.md
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/README.md
rename to tika-server/tika-server-grpc/README.md
diff --git a/tika-pipes/tika-pipes-grpc/corpa-files/014760.docx b/tika-server/tika-server-grpc/corpa-files/014760.docx
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/corpa-files/014760.docx
rename to tika-server/tika-server-grpc/corpa-files/014760.docx
diff --git a/tika-pipes/tika-pipes-grpc/corpa-files/017091.docx b/tika-server/tika-server-grpc/corpa-files/017091.docx
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/corpa-files/017091.docx
rename to tika-server/tika-server-grpc/corpa-files/017091.docx
diff --git a/tika-pipes/tika-pipes-grpc/corpa-files/017097.docx b/tika-server/tika-server-grpc/corpa-files/017097.docx
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/corpa-files/017097.docx
rename to tika-server/tika-server-grpc/corpa-files/017097.docx
diff --git a/tika-pipes/tika-pipes-grpc/corpa-files/018367.docx b/tika-server/tika-server-grpc/corpa-files/018367.docx
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/corpa-files/018367.docx
rename to tika-server/tika-server-grpc/corpa-files/018367.docx
diff --git a/tika-pipes/tika-pipes-grpc/corpa-files/018370.docx b/tika-server/tika-server-grpc/corpa-files/018370.docx
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/corpa-files/018370.docx
rename to tika-server/tika-server-grpc/corpa-files/018370.docx
diff --git a/tika-pipes/tika-pipes-grpc/corpa-files/018371.docx b/tika-server/tika-server-grpc/corpa-files/018371.docx
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/corpa-files/018371.docx
rename to tika-server/tika-server-grpc/corpa-files/018371.docx
diff --git a/tika-pipes/tika-pipes-grpc/corpa-files/021524.docx b/tika-server/tika-server-grpc/corpa-files/021524.docx
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/corpa-files/021524.docx
rename to tika-server/tika-server-grpc/corpa-files/021524.docx
diff --git a/tika-pipes/tika-pipes-grpc/corpa-files/file-that-confuses-tika.log b/tika-server/tika-server-grpc/corpa-files/file-that-confuses-tika.log
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/corpa-files/file-that-confuses-tika.log
rename to tika-server/tika-server-grpc/corpa-files/file-that-confuses-tika.log
diff --git a/tika-pipes/tika-pipes-grpc/docker-build/Dockerfile b/tika-server/tika-server-grpc/docker-build/Dockerfile
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/docker-build/Dockerfile
rename to tika-server/tika-server-grpc/docker-build/Dockerfile
diff --git a/tika-pipes/tika-pipes-grpc/docker-build/docker-build.sh b/tika-server/tika-server-grpc/docker-build/docker-build.sh
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/docker-build/docker-build.sh
rename to tika-server/tika-server-grpc/docker-build/docker-build.sh
diff --git a/tika-pipes/tika-pipes-grpc/docker-build/start-tika-grpc.sh b/tika-server/tika-server-grpc/docker-build/start-tika-grpc.sh
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/docker-build/start-tika-grpc.sh
rename to tika-server/tika-server-grpc/docker-build/start-tika-grpc.sh
diff --git a/tika-pipes/tika-pipes-grpc/pom.xml b/tika-server/tika-server-grpc/pom.xml
similarity index 99%
rename from tika-pipes/tika-pipes-grpc/pom.xml
rename to tika-server/tika-server-grpc/pom.xml
index d2d561af5..1f24b5e59 100644
--- a/tika-pipes/tika-pipes-grpc/pom.xml
+++ b/tika-server/tika-server-grpc/pom.xml
@@ -4,11 +4,11 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.tika</groupId>
- <artifactId>tika-pipes</artifactId>
+ <artifactId>tika-server</artifactId>
<version>4.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>tika-pipes-grpc</artifactId>
+ <artifactId>tika-server-grpc</artifactId>
<name>Tika Pipes GRPC Server</name>
<description>Tika Pipes GRPC Server provides bidirectional streams to and from the Tika parsers for
running parsers on files in a platform independent and efficient way using HTTP2.
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/TikaPipesApplication.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/TikaPipesApplication.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/TikaPipesApplication.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/TikaPipesApplication.java
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerCustomizer.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerCustomizer.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerCustomizer.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerCustomizer.java
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcService.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcService.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcService.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcService.java
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/job/JobStatus.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/job/JobStatus.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/job/JobStatus.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/job/JobStatus.java
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/model/FetchAndParseStatus.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/model/FetchAndParseStatus.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/model/FetchAndParseStatus.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/model/FetchAndParseStatus.java
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/parser/TikaPipesApplicationConfiguration.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/parser/TikaPipesApplicationConfiguration.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/parser/TikaPipesApplicationConfiguration.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/parser/TikaPipesApplicationConfiguration.java
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/plugin/ClasspathPluginPropertiesFinder.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/plugin/ClasspathPluginPropertiesFinder.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/plugin/ClasspathPluginPropertiesFinder.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/plugin/ClasspathPluginPropertiesFinder.java
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/plugin/GrpcPluginManager.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/plugin/GrpcPluginManager.java
similarity index 93%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/plugin/GrpcPluginManager.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/plugin/GrpcPluginManager.java
index 818c48268..999b1f9f7 100644
--- a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/plugin/GrpcPluginManager.java
+++ b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/plugin/GrpcPluginManager.java
@@ -6,7 +6,12 @@ import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
-import org.pf4j.*;
+import org.pf4j.DefaultPluginManager;
+import org.pf4j.PluginDescriptor;
+import org.pf4j.PluginDescriptorFinder;
+import org.pf4j.PluginLoader;
+import org.pf4j.PluginRepository;
+import org.pf4j.PluginWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/plugin/PluginConfig.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/plugin/PluginConfig.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/plugin/PluginConfig.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/plugin/PluginConfig.java
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/repo/EmitterRepository.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/repo/EmitterRepository.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/repo/EmitterRepository.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/repo/EmitterRepository.java
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/repo/FetcherRepository.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/repo/FetcherRepository.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/repo/FetcherRepository.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/repo/FetcherRepository.java
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/repo/IgniteRepositoryConfiguration.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/repo/IgniteRepositoryConfiguration.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/repo/IgniteRepositoryConfiguration.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/repo/IgniteRepositoryConfiguration.java
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/repo/JobStatusRepository.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/repo/JobStatusRepository.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/repo/JobStatusRepository.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/repo/JobStatusRepository.java
diff --git a/tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/repo/PipeIteratorRepository.java b/tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/repo/PipeIteratorRepository.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/java/org/apache/tika/pipes/repo/PipeIteratorRepository.java
rename to tika-server/tika-server-grpc/src/main/java/org/apache/tika/pipes/repo/PipeIteratorRepository.java
diff --git a/tika-pipes/tika-pipes-grpc/src/main/resources/application.yaml b/tika-server/tika-server-grpc/src/main/resources/application.yaml
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/main/resources/application.yaml
rename to tika-server/tika-server-grpc/src/main/resources/application.yaml
diff --git a/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/TikaPipesIntegrationTestBase.java b/tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/TikaPipesIntegrationTestBase.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/TikaPipesIntegrationTestBase.java
rename to tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/TikaPipesIntegrationTestBase.java
diff --git a/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerEmittersCrudTest.java b/tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerEmittersCrudTest.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerEmittersCrudTest.java
rename to tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerEmittersCrudTest.java
diff --git a/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerFetchAndParseTest.java b/tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerFetchAndParseTest.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerFetchAndParseTest.java
rename to tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerFetchAndParseTest.java
diff --git a/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerFetchersCrudTest.java b/tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerFetchersCrudTest.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerFetchersCrudTest.java
rename to tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerFetchersCrudTest.java
diff --git a/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerJsonSchemaTest.java b/tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerJsonSchemaTest.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerJsonSchemaTest.java
rename to tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerJsonSchemaTest.java
diff --git a/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerPipeIteratorsCrudTest.java b/tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerPipeIteratorsCrudTest.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerPipeIteratorsCrudTest.java
rename to tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerPipeIteratorsCrudTest.java
diff --git a/tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerPipeJobTest.java b/tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerPipeJobTest.java
similarity index 100%
rename from tika-pipes/tika-pipes-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerPipeJobTest.java
rename to tika-server/tika-server-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerPipeJobTest.java
diff --git a/tika-server/tika-server-spring/pom.xml b/tika-server/tika-server-spring/pom.xml
index 5fbd2c67b..2e4a7bea1 100644
--- a/tika-server/tika-server-spring/pom.xml
+++ b/tika-server/tika-server-spring/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.tika</groupId>
- <artifactId>tika-parent</artifactId>
+ <artifactId>tika-server</artifactId>
<version>4.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -110,6 +110,12 @@
<groupId>org.apache.tika</groupId>
<artifactId>tika-pipes-core</artifactId>
<version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.jetbrains</groupId>
+ <artifactId>annotations</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
@@ -125,6 +131,12 @@
<version>${springdoc.version}</version>
</dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-langdetect-optimaize</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<!-- WebJars Locator for resolving WebJars resources -->
<dependency>
<groupId>org.webjars</groupId>
@@ -149,6 +161,13 @@
<version>0.2.6</version>
</dependency>
+ <!-- Jakarta JAX-RS API for MultivaluedMap -->
+ <dependency>
+ <groupId>jakarta.ws.rs</groupId>
+ <artifactId>jakarta.ws.rs-api</artifactId>
+ <version>3.1.0</version>
+ </dependency>
+
<!-- Test Dependencies -->
<dependency>
<groupId>org.springframework.boot</groupId>
diff --git a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/InformationServicesController.java b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/InformationServicesController.java
deleted file mode 100644
index 2816c1f11..000000000
--- a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/InformationServicesController.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.controller;
-
-import java.util.Map;
-
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.RestController;
-
-import org.apache.tika.server.api.InformationServicesApi;
-import org.apache.tika.server.model.DefaultDetector;
-import org.apache.tika.server.model.DetailedParsers;
-import org.apache.tika.server.model.GetMimetypeDetails200Response;
-import org.apache.tika.server.model.Parsers;
-
-/**
- * Controller for utility information services.
- * Handles endpoints for available parsers, detectors, mime types, etc.
- * Covers Information Services tag.
- */
-@RestController
-public class InformationServicesController implements InformationServicesApi {
-
- @Override
- public ResponseEntity<String> getEndpoints() {
- // TODO: Implement endpoint listing
- return InformationServicesApi.super.getEndpoints();
- }
-
- @Override
- public ResponseEntity<DefaultDetector> getDetectors() {
- // TODO: Implement detector information
- return InformationServicesApi.super.getDetectors();
- }
-
- @Override
- public ResponseEntity<Map<String, Object>> getMimetypes() {
- // TODO: Implement mime types listing
- return InformationServicesApi.super.getMimetypes();
- }
-
- @Override
- public ResponseEntity<GetMimetypeDetails200Response> getMimetypeDetails(String type, String subtype) {
- // TODO: Implement specific mime type details
- return InformationServicesApi.super.getMimetypeDetails(type, subtype);
- }
-
- @Override
- public ResponseEntity<Parsers> getParsers() {
- // TODO: Implement parsers listing
- return InformationServicesApi.super.getParsers();
- }
-
- @Override
- public ResponseEntity<DetailedParsers> getParsersDetails() {
- // TODO: Implement detailed parsers information
- return InformationServicesApi.super.getParsersDetails();
- }
-}
diff --git a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/LanguageResourceController.java b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/LanguageResourceController.java
index ed0bc10da..09699fadf 100644
--- a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/LanguageResourceController.java
+++ b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/LanguageResourceController.java
@@ -16,14 +16,20 @@
*/
package org.apache.tika.server.controller;
-import java.util.Optional;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.io.IOUtils;
import org.springframework.core.io.Resource;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.RestController;
-import org.springframework.web.context.request.NativeWebRequest;
+import org.apache.tika.langdetect.optimaize.OptimaizeLangDetector;
+import org.apache.tika.language.detect.LanguageResult;
import org.apache.tika.server.api.LanguageResourceApi;
+import org.apache.tika.server.exception.TikaServerRuntimeException;
/**
* Controller for language identification services.
@@ -31,28 +37,37 @@ import org.apache.tika.server.api.LanguageResourceApi;
*/
@RestController
public class LanguageResourceController implements LanguageResourceApi {
- @Override
- public Optional<NativeWebRequest> getRequest() {
- return LanguageResourceApi.super.getRequest();
- }
-
@Override
public ResponseEntity<String> postLanguageStream(Resource body) {
- return LanguageResourceApi.super.postLanguageStream(body);
+ try {
+ InputStream is = body.getInputStream();
+ String fileTxt = IOUtils.toString(is, UTF_8);
+ LanguageResult language = new OptimaizeLangDetector()
+ .loadModels()
+ .detect(fileTxt);
+ String detectedLang = language.getLanguage();
+ return ResponseEntity.ok(detectedLang);
+ } catch (IOException e) {
+ throw new TikaServerRuntimeException(e);
+ }
}
@Override
public ResponseEntity<String> postLanguageString(String body) {
- return LanguageResourceApi.super.postLanguageString(body);
+ LanguageResult language = new OptimaizeLangDetector()
+ .loadModels()
+ .detect(body);
+ String detectedLang = language.getLanguage();
+ return ResponseEntity.ok(detectedLang);
}
@Override
public ResponseEntity<String> putLanguageStream(Resource body) {
- return LanguageResourceApi.super.putLanguageStream(body);
+ return postLanguageStream(body);
}
@Override
public ResponseEntity<String> putLanguageString(String body) {
- return LanguageResourceApi.super.putLanguageString(body);
+ return postLanguageString(body);
}
}
diff --git a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/MetadataController.java b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/MetadataController.java
index ad0d5fbab..3b931040c 100644
--- a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/MetadataController.java
+++ b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/MetadataController.java
@@ -16,23 +16,45 @@
*/
package org.apache.tika.server.controller;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
+import jakarta.servlet.http.HttpServletRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.NativeWebRequest;
import org.springframework.web.multipart.MultipartFile;
+import org.apache.tika.metadata.Metadata;
import org.apache.tika.server.api.MetadataResourceApi;
+import org.apache.tika.server.service.TikaMetadataFillerService;
/**
* Controller for metadata extraction services.
* Handles the /meta and /xmpmeta endpoints for Metadata Resource and XMP Metadata Resource tags.
+ * Ported from the legacy JAX-RS MetadataResource implementation.
*/
@RestController
public class MetadataController implements MetadataResourceApi {
+ private static final Logger LOG = LoggerFactory.getLogger(MetadataController.class);
+
+ @Autowired
+ private TikaMetadataFillerService tikaMetadataFillerService;
+
@Override
public Optional<NativeWebRequest> getRequest() {
return MetadataResourceApi.super.getRequest();
@@ -40,16 +62,210 @@ public class MetadataController implements MetadataResourceApi {
@Override
public ResponseEntity<Map<String, String>> postDocumentMetaForm(MultipartFile file) {
- return MetadataResourceApi.super.postDocumentMetaForm(file);
+ try {
+ Metadata metadata = new Metadata();
+ Map<String, String> headers = new HashMap<>();
+ // Convert MultipartFile headers if available
+ if (file.getOriginalFilename() != null) {
+ headers.put("Content-Disposition", "filename=" + file.getOriginalFilename());
+ }
+ if (file.getContentType() != null) {
+ headers.put("Content-Type", file.getContentType());
+ }
+
+ Metadata result = parseMetadata(file.getInputStream(), metadata, headers);
+ return ResponseEntity.ok(metadataToMap(result));
+ } catch (Exception e) {
+ LOG.error("Failed to process multipart form metadata extraction", e);
+ return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
+ }
}
@Override
public ResponseEntity<Map<String, String>> putDocumentGetMetaValue(String metadataKey, Resource body) {
- return MetadataResourceApi.super.putDocumentGetMetaValue(metadataKey, body);
+ try {
+ Metadata metadata = new Metadata();
+ Map<String, String> headers = new HashMap<>();
+
+ boolean success = false;
+ try {
+ parseMetadata(body.getInputStream(), metadata, headers);
+ success = true;
+ } catch (Exception e) {
+ LOG.info("Failed to process field {}", metadataKey, e);
+ }
+
+ if (!success || metadata.get(metadataKey) == null) {
+ return ResponseEntity.status(success ? HttpStatus.NOT_FOUND : HttpStatus.BAD_REQUEST).build();
+ }
+
+ // Remove fields we don't care about for the response
+ Metadata filteredMetadata = new Metadata();
+ String[] values = metadata.getValues(metadataKey);
+ for (String value : values) {
+ filteredMetadata.add(metadataKey, value);
+ }
+
+ return ResponseEntity.ok(metadataToMap(filteredMetadata));
+ } catch (Exception e) {
+ LOG.error("Failed to extract metadata field: " + metadataKey, e);
+ return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
+ }
}
@Override
public ResponseEntity<Map<String, String>> putDocumentMeta(Resource body) {
- return MetadataResourceApi.super.putDocumentMeta(body);
+ try {
+ Metadata metadata = new Metadata();
+ Map<String, String> headers = new HashMap<>();
+
+ Metadata result = parseMetadata(body.getInputStream(), metadata, headers);
+ return ResponseEntity.ok(metadataToMap(result));
+ } catch (Exception e) {
+ LOG.error("Failed to extract metadata", e);
+ return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * Additional Spring Boot endpoints to handle the legacy API patterns
+ */
+ @PutMapping(value = "/meta", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.TEXT_PLAIN_VALUE})
+ public ResponseEntity<?> getMetadata(@RequestBody Resource body,
+ @RequestHeader HttpHeaders httpHeaders,
+ @RequestHeader(value = "Accept", defaultValue = MediaType.APPLICATION_JSON_VALUE) String accept,
+ HttpServletRequest request) throws Exception {
+ Metadata metadata = new Metadata();
+ Map<String, String> headerMap = convertHeaders(httpHeaders);
+
+ Metadata result = parseMetadata(body.getInputStream(), metadata, headerMap);
+
+ if (accept.contains(MediaType.TEXT_PLAIN_VALUE)) {
+ return ResponseEntity.ok()
+ .contentType(MediaType.TEXT_PLAIN)
+ .body(metadataToText(result));
+ } else {
+ return ResponseEntity.ok()
+ .contentType(MediaType.APPLICATION_JSON)
+ .body(metadataToMap(result));
+ }
+ }
+
+ @PutMapping(value = "/meta/{field}", produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.TEXT_PLAIN_VALUE})
+ public ResponseEntity<?> getMetadataField(@RequestBody Resource body,
+ @PathVariable String field,
+ @RequestHeader HttpHeaders httpHeaders,
+ @RequestHeader(value = "Accept", defaultValue = MediaType.APPLICATION_JSON_VALUE) String accept,
+ HttpServletRequest request) throws Exception {
+
+ HttpStatus defaultErrorResponse = HttpStatus.BAD_REQUEST;
+ Metadata metadata = new Metadata();
+ Map<String, String> headerMap = convertHeaders(httpHeaders);
+
+ boolean success = false;
+ try {
+ parseMetadata(body.getInputStream(), metadata, headerMap);
+ defaultErrorResponse = HttpStatus.NOT_FOUND;
+ success = true;
+ } catch (Exception e) {
+ LOG.info("Failed to process field {}", field, e);
+ }
+
+ if (!success || metadata.get(field) == null) {
+ return ResponseEntity.status(defaultErrorResponse)
+ .body("Failed to get metadata field " + field);
+ }
+
+ // Remove fields we don't care about for the response
+ Metadata filteredMetadata = new Metadata();
+ String[] values = metadata.getValues(field);
+ for (String value : values) {
+ filteredMetadata.add(field, value);
+ }
+
+ if (accept.contains(MediaType.TEXT_PLAIN_VALUE)) {
+ String value = filteredMetadata.get(field);
+ return ResponseEntity.ok()
+ .contentType(MediaType.TEXT_PLAIN)
+ .body(value != null ? value : "");
+ } else {
+ return ResponseEntity.ok()
+ .contentType(MediaType.APPLICATION_JSON)
+ .body(metadataToMap(filteredMetadata));
+ }
+ }
+
+ /**
+ * Core metadata parsing method ported from legacy implementation
+ */
+ protected Metadata parseMetadata(InputStream is, Metadata metadata, Map<String, String> headers) throws IOException {
+// final ParseContext context = new ParseContext();
+// Parser parser = TikaResource.createParser();
+//
+// MultiValueMap<String, String> multiValueHeaders = new org.springframework.util.LinkedMultiValueMap<>();
+// for (Map.Entry<String, String> entry : headers.entrySet()) {
+// multiValueHeaders.add(entry.getKey(), entry.getValue());
+// }
+//
+// tikaMetadataFillerService.fillMetadata(parser, metadata, multiValueHeaders);
+// fillParseContext(multiValueHeaders, metadata, context);
+//
+// // No need to parse embedded docs
+// context.set(DocumentSelector.class, metadata1 -> false);
+//
+// TikaResource.logRequest(LOG, "/meta", metadata);
+// TikaResource.parse(parser, LOG, "/meta", is, new LanguageHandler() {
+// public void endDocument() {
+// String language = getLanguage().getLanguage();
+// if (language != null) {
+// metadata.set("language", language);
+// }
+// }
+// }, metadata, context);
+// return metadata;
+ return null;
+ }
+
+ /**
+ * Convert Metadata to Map for JSON responses
+ */
+ private Map<String, String> metadataToMap(Metadata metadata) {
+ Map<String, String> map = new HashMap<>();
+ for (String name : metadata.names()) {
+ String[] values = metadata.getValues(name);
+ if (values.length == 1) {
+ map.put(name, values[0]);
+ } else if (values.length > 1) {
+ map.put(name, String.join(", ", values));
+ }
+ }
+ return map;
+ }
+
+ /**
+ * Convert Metadata to text for plain text responses
+ */
+ private String metadataToText(Metadata metadata) {
+ StringBuilder sb = new StringBuilder();
+ for (String name : metadata.names()) {
+ String[] values = metadata.getValues(name);
+ for (String value : values) {
+ sb.append(name).append(": ").append(value).append("\n");
+ }
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Convert Spring HttpHeaders to Map
+ */
+ private Map<String, String> convertHeaders(HttpHeaders httpHeaders) {
+ Map<String, String> headerMap = new HashMap<>();
+ for (Map.Entry<String, java.util.List<String>> entry : httpHeaders.entrySet()) {
+ if (!entry.getValue().isEmpty()) {
+ headerMap.put(entry.getKey(), entry.getValue().get(0));
+ }
+ }
+ return headerMap;
}
}
diff --git a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/exception/TikaServerRuntimeException.java b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/exception/TikaServerRuntimeException.java
new file mode 100644
index 000000000..b0335f556
--- /dev/null
+++ b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/exception/TikaServerRuntimeException.java
@@ -0,0 +1,35 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ *
+ */
+
+package org.apache.tika.server.exception;
+
+public class TikaServerRuntimeException extends RuntimeException {
+ public TikaServerRuntimeException(String message) {
+ super(message);
+ }
+
+ public TikaServerRuntimeException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public TikaServerRuntimeException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/service/ParserService.java b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/service/ParserService.java
new file mode 100644
index 000000000..01e0d5575
--- /dev/null
+++ b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/service/ParserService.java
@@ -0,0 +1,48 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ *
+ */
+
+package org.apache.tika.server.service;
+
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.DigestingParser;
+import org.apache.tika.parser.Parser;
+
+public class ParserService {
+
+
+ @SuppressWarnings("serial")
+ public static Parser createParser() {
+ final Parser parser = new AutoDetectParser(TIKA_CONFIG);
+
+ if (DIGESTER != null) {
+ boolean skipContainer = false;
+ if (TIKA_CONFIG
+ .getAutoDetectParserConfig()
+ .getDigesterFactory() != null && TIKA_CONFIG
+ .getAutoDetectParserConfig()
+ .getDigesterFactory()
+ .isSkipContainerDocument()) {
+ skipContainer = true;
+ }
+ return new DigestingParser(parser, DIGESTER, skipContainer);
+ }
+ return parser;
+ }
+}
diff --git a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/service/TikaMetadataFillerService.java b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/service/TikaMetadataFillerService.java
new file mode 100644
index 000000000..5f543d732
--- /dev/null
+++ b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/service/TikaMetadataFillerService.java
@@ -0,0 +1,124 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ *
+ */
+
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ *
+ */
+
+package org.apache.tika.server.service;
+
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.http.HttpHeaders;
+import org.springframework.stereotype.Component;
+import org.springframework.util.MultiValueMap;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.Parser;
+
+@Component
+public class TikaMetadataFillerService {
+ private static final String META_PREFIX = "X-Tika-Meta-";
+
+ @SuppressWarnings("serial")
+ public void fillMetadata(Parser parser, Metadata metadata, MultiValueMap<String, String> httpHeaders) {
+ String fileName = detectFilename(httpHeaders);
+ if (fileName != null) {
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, fileName);
+ }
+
+ String contentTypeHeader = httpHeaders.getFirst(HttpHeaders.CONTENT_TYPE);
+ jakarta.ws.rs.core.MediaType mediaType = (contentTypeHeader == null || "*/*".equals(contentTypeHeader)) ? null : jakarta.ws.rs.core.MediaType.valueOf(contentTypeHeader);
+ if (mediaType != null && "xml".equals(mediaType.getSubtype())) {
+ mediaType = null;
+ }
+
+ if (mediaType != null && mediaType.equals(jakarta.ws.rs.core.MediaType.APPLICATION_OCTET_STREAM_TYPE)) {
+ mediaType = null;
+ }
+
+ if (mediaType != null) {
+ metadata.add(Metadata.CONTENT_TYPE, mediaType.toString());
+ metadata.add(TikaCoreProperties.CONTENT_TYPE_USER_OVERRIDE, mediaType.toString());
+ }
+
+ if (httpHeaders.containsKey("Content-Length")) {
+ metadata.set(Metadata.CONTENT_LENGTH, httpHeaders.getFirst("Content-Length"));
+ }
+
+ for (Map.Entry<String, List<String>> e : httpHeaders.entrySet()) {
+ if (e
+ .getKey()
+ .startsWith(META_PREFIX)) {
+ String tikaKey = e
+ .getKey()
+ .substring(META_PREFIX.length());
+ for (String value : e.getValue()) {
+ metadata.add(tikaKey, value);
+ }
+ }
+ }
+ }
+
+ private String detectFilename(MultiValueMap<String, String> httpHeaders) {
+ String disposition = httpHeaders.getFirst("Content-Disposition");
+ if (disposition != null) {
+ // Parse filename from Content-Disposition header
+ String[] parts = disposition.split(";");
+ for (String part : parts) {
+ part = part.trim();
+ if (part.startsWith("filename=")) {
+ String filename = part.substring("filename=".length());
+ // Remove surrounding quotes if present
+ if (filename.startsWith("\"") && filename.endsWith("\"")) {
+ filename = filename.substring(1, filename.length() - 1);
+ }
+ return filename;
+ }
+ }
+ }
+
+ // Check for X-Tika-OCRTesseractPath or similar headers that might contain filename
+ String resourceName = httpHeaders.getFirst(TikaCoreProperties.RESOURCE_NAME_KEY.getName());
+ if (resourceName != null) {
+ return resourceName;
+ }
+
+ return null;
+ }
+}
diff --git a/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/InformationServicesControllerIntegrationTest.java b/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/InformationServicesControllerIntegrationTest.java
deleted file mode 100644
index af5b0f94f..000000000
--- a/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/InformationServicesControllerIntegrationTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.controller;
-
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.server.IntegrationTestBase;
-
-/**
- * Integration tests for InformationServicesController.
- * Tests utility information service endpoints.
- */
-public class InformationServicesControllerIntegrationTest extends IntegrationTestBase {
-
- @Test
- public void testPlaceholder() {
- // TODO: Implement integration tests for InformationServicesController
- // - Test GET / for endpoint listing
- // - Test GET /detectors for detector information
- // - Test GET /mime-types for mime type listing
- // - Test GET /mime-types/{type}/{subtype} for specific mime type details
- // - Test GET /parsers for parser listing
- // - Test GET /parsers/details for detailed parser information
- }
-}
diff --git a/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/LanguageResourceControllerIntegrationTest.java b/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/LanguageResourceControllerIntegrationTest.java
index ae4e68975..5b6bfa807 100644
--- a/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/LanguageResourceControllerIntegrationTest.java
+++ b/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/LanguageResourceControllerIntegrationTest.java
@@ -16,21 +16,116 @@
*/
package org.apache.tika.server.controller;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+import java.io.InputStream;
+
import org.junit.jupiter.api.Test;
+import org.springframework.http.MediaType;
import org.apache.tika.server.IntegrationTestBase;
/**
* Integration tests for LanguageResourceController.
* Tests language identification service endpoints.
+ * Ported from the legacy JAX-RS implementation.
*/
public class LanguageResourceControllerIntegrationTest extends IntegrationTestBase {
+ private static final String LANG_PATH = "/language";
+ private static final String LANG_STREAM_PATH = LANG_PATH + "/stream";
+ private static final String LANG_STRING_PATH = LANG_PATH + "/string";
+ private static final String ENGLISH_STRING = "This is English!";
+ private static final String FRENCH_STRING = "c'est comme ci comme ça";
+
+ @Test
+ public void testDetectEnglishString() throws Exception {
+ mockMvc.perform(put(LANG_STRING_PATH)
+ .contentType(MediaType.TEXT_PLAIN)
+ .accept(MediaType.TEXT_PLAIN)
+ .content(ENGLISH_STRING))
+ .andExpect(status().isOk())
+ .andExpect(content().string("en"));
+ }
+
+ @Test
+ public void testDetectFrenchString() throws Exception {
+ mockMvc.perform(put(LANG_STRING_PATH)
+ .contentType(MediaType.TEXT_PLAIN)
+ .accept(MediaType.TEXT_PLAIN)
+ .content(FRENCH_STRING))
+ .andExpect(status().isOk())
+ .andExpect(content().string("fr"));
+ }
+
+ @Test
+ public void testDetectEnglishFile() throws Exception {
+ InputStream englishStream = getClass().getResourceAsStream("/test-documents/english.txt");
+
+ mockMvc.perform(put(LANG_STREAM_PATH)
+ .contentType(MediaType.TEXT_PLAIN)
+ .accept(MediaType.TEXT_PLAIN)
+ .content(englishStream.readAllBytes()))
+ .andExpect(status().isOk())
+ .andExpect(content().string("en"));
+ }
+
+ @Test
+ public void testDetectFrenchFile() throws Exception {
+ InputStream frenchStream = getClass().getResourceAsStream("/test-documents/french.txt");
+
+ mockMvc.perform(put(LANG_STREAM_PATH)
+ .contentType(MediaType.TEXT_PLAIN)
+ .accept(MediaType.TEXT_PLAIN)
+ .content(frenchStream.readAllBytes()))
+ .andExpect(status().isOk())
+ .andExpect(content().string("fr"));
+ }
+
+ @Test
+ public void testDetectEnglishStringPost() throws Exception {
+ mockMvc.perform(post(LANG_STRING_PATH)
+ .contentType(MediaType.TEXT_PLAIN)
+ .accept(MediaType.TEXT_PLAIN)
+ .content(ENGLISH_STRING))
+ .andExpect(status().isOk())
+ .andExpect(content().string("en"));
+ }
+
+ @Test
+ public void testDetectFrenchStringPost() throws Exception {
+ mockMvc.perform(post(LANG_STRING_PATH)
+ .contentType(MediaType.TEXT_PLAIN)
+ .accept(MediaType.TEXT_PLAIN)
+ .content(FRENCH_STRING))
+ .andExpect(status().isOk())
+ .andExpect(content().string("fr"));
+ }
+
@Test
- public void testPlaceholder() {
- // TODO: Implement integration tests for LanguageResourceController
- // - Test POST/PUT /language/stream for UTF-8 text file language identification
- // - Test POST/PUT /language/string for text string language identification
- // - Test language detection accuracy with various languages
+ public void testDetectEnglishFilePost() throws Exception {
+ InputStream englishStream = getClass().getResourceAsStream("/test-documents/english.txt");
+
+ mockMvc.perform(post(LANG_STREAM_PATH)
+ .contentType(MediaType.TEXT_PLAIN)
+ .accept(MediaType.TEXT_PLAIN)
+ .content(englishStream.readAllBytes()))
+ .andExpect(status().isOk())
+ .andExpect(content().string("en"));
+ }
+
+ @Test
+ public void testDetectFrenchFilePost() throws Exception {
+ InputStream frenchStream = getClass().getResourceAsStream("/test-documents/french.txt");
+
+ mockMvc.perform(post(LANG_STREAM_PATH)
+ .contentType(MediaType.TEXT_PLAIN)
+ .accept(MediaType.TEXT_PLAIN)
+ .content(frenchStream.readAllBytes()))
+ .andExpect(status().isOk())
+ .andExpect(content().string("fr"));
}
}
diff --git a/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/MetadataControllerIntegrationTest.java b/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/MetadataControllerIntegrationTest.java
index bf2354bc5..c1854c046 100644
--- a/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/MetadataControllerIntegrationTest.java
+++ b/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/MetadataControllerIntegrationTest.java
@@ -16,24 +16,229 @@
*/
package org.apache.tika.server.controller;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+
import org.junit.jupiter.api.Test;
+import org.springframework.http.MediaType;
+import org.springframework.test.web.servlet.MvcResult;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.serialization.JsonMetadata;
import org.apache.tika.server.IntegrationTestBase;
/**
* Integration tests for MetadataController.
* Tests metadata extraction service endpoints.
+ * Ported from the legacy JAX-RS implementation and adapted for Spring Boot supported content types.
*/
public class MetadataControllerIntegrationTest extends IntegrationTestBase {
+ private static final String META_PATH = "/meta";
+ private static final String TEST_DOC = "/test-documents/test.doc";
+ private static final String TEST_PASSWORD_PROTECTED = "/test-documents/password-protected.doc";
+
+ @Test
+ public void testSimpleWordJSON() throws Exception {
+ InputStream docStream = getClass().getResourceAsStream(TEST_DOC);
+
+ MvcResult result = mockMvc.perform(put(META_PATH)
+ .contentType("application/msword")
+ .accept("application/json")
+ .content(docStream.readAllBytes()))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String jsonContent = result.getResponse().getContentAsString();
+ Reader reader = new InputStreamReader(new ByteArrayInputStream(jsonContent.getBytes(UTF_8)), UTF_8);
+
+ Metadata metadata = JsonMetadata.fromJson(reader);
+
+ // Basic validation - check that we have some metadata
+ assertNotNull(metadata);
+ assertTrue(metadata.names().length > 0);
+
+ // Check for common metadata fields that should be present
+ String contentType = metadata.get("Content-Type");
+ assertNotNull(contentType, "Content-Type should be present");
+ assertTrue(contentType.contains("application") || contentType.contains("msword"),
+ "Content type should indicate Word document");
+ }
+
+ @Test
+ public void testSimpleWordText() throws Exception {
+ InputStream docStream = getClass().getResourceAsStream(TEST_DOC);
+
+ MvcResult result = mockMvc.perform(put(META_PATH)
+ .contentType("application/msword")
+ .accept("text/plain")
+ .content(docStream.readAllBytes()))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String textContent = result.getResponse().getContentAsString();
+ assertNotNull(textContent);
+ assertTrue(textContent.length() > 0, "Should return some text content");
+ }
+
+ @Test
+ public void testPasswordProtected() throws Exception {
+ InputStream docStream = getClass().getResourceAsStream(TEST_PASSWORD_PROTECTED);
+
+ // Won't work, no password given
+ mockMvc.perform(put(META_PATH)
+ .contentType("application/msword")
+ .accept("application/json")
+ .content(docStream.readAllBytes()))
+ .andExpect(status().is5xxServerError());
+
+ // Try again, this time with the wrong password
+ docStream = getClass().getResourceAsStream(TEST_PASSWORD_PROTECTED);
+ mockMvc.perform(put(META_PATH)
+ .contentType("application/msword")
+ .accept("application/json")
+ .header("Password", "wrong password")
+ .content(docStream.readAllBytes()))
+ .andExpect(status().is5xxServerError());
+
+ // Try again, this time with the correct password
+ docStream = getClass().getResourceAsStream(TEST_PASSWORD_PROTECTED);
+ MvcResult result = mockMvc.perform(put(META_PATH)
+ .contentType("application/msword")
+ .accept("application/json")
+ .header("Password", "tika")
+ .content(docStream.readAllBytes()))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ // Check results
+ String jsonContent = result.getResponse().getContentAsString();
+ Reader reader = new InputStreamReader(new ByteArrayInputStream(jsonContent.getBytes(UTF_8)), UTF_8);
+
+ Metadata metadata = JsonMetadata.fromJson(reader);
+ assertNotNull(metadata);
+ assertTrue(metadata.names().length > 0, "Should have extracted metadata with correct password");
+ }
+
+ @Test
+ public void testJSON() throws Exception {
+ InputStream docStream = getClass().getResourceAsStream(TEST_DOC);
+
+ MvcResult result = mockMvc.perform(put(META_PATH)
+ .contentType("application/msword")
+ .accept("application/json")
+ .content(docStream.readAllBytes()))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String jsonContent = result.getResponse().getContentAsString();
+ Reader reader = new InputStreamReader(new ByteArrayInputStream(jsonContent.getBytes(UTF_8)), UTF_8);
+
+ Metadata metadata = JsonMetadata.fromJson(reader);
+ assertNotNull(metadata);
+ assertTrue(metadata.names().length > 0, "Should have extracted metadata");
+ }
+
+ @Test
+ public void testGetField_XXX_NotFound() throws Exception {
+ InputStream docStream = getClass().getResourceAsStream(TEST_DOC);
+
+ mockMvc.perform(put(META_PATH + "/xxx")
+ .contentType("application/msword")
+ .accept(MediaType.APPLICATION_JSON_VALUE)
+ .content(docStream.readAllBytes()))
+ .andExpect(status().isNotFound());
+ }
+
@Test
- public void testPlaceholder() {
- // TODO: Implement integration tests for MetadataController
- // - Test PUT /meta for metadata extraction
- // - Test PUT /meta/{metadata_key} for specific metadata key extraction
- // - Test POST /meta/form for multipart form metadata extraction
- // - Test PUT /xmpmeta for XMP metadata extraction in RDF/XML format
- // - Test PUT /xmpmeta/{metadata_key} for specific XMP metadata key extraction
- // - Test POST /xmpmeta/form for multipart form XMP metadata extraction
+ public void testGetField_ContentType_JSON_Found() throws Exception {
+ InputStream docStream = getClass().getResourceAsStream(TEST_DOC);
+
+ MvcResult result = mockMvc.perform(put(META_PATH + "/Content-Type")
+ .contentType("application/msword")
+ .accept(MediaType.APPLICATION_JSON_VALUE)
+ .content(docStream.readAllBytes()))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String jsonContent = result.getResponse().getContentAsString();
+ Reader reader = new InputStreamReader(new ByteArrayInputStream(jsonContent.getBytes(UTF_8)), UTF_8);
+
+ Metadata metadata = JsonMetadata.fromJson(reader);
+ assertNotNull(metadata.get("Content-Type"));
+ assertEquals(1, metadata.names().length, "Should only return the requested field");
+ }
+
+ @Test
+ public void testGetField_Author_TEXT_Partial_BAD_REQUEST() throws Exception {
+ InputStream docStream = getClass().getResourceAsStream(TEST_DOC);
+ byte[] partialContent = copyBytes(docStream, 8000);
+
+ mockMvc.perform(put(META_PATH + "/Author")
+ .contentType("application/msword")
+ .accept(MediaType.TEXT_PLAIN_VALUE)
+ .content(partialContent))
+ .andExpect(status().isBadRequest());
+ }
+
+ @Test
+ public void testGetField_ContentType_TEXT_Partial_Found() throws Exception {
+ InputStream docStream = getClass().getResourceAsStream(TEST_DOC);
+ byte[] partialContent = copyBytes(docStream, 12000);
+
+ MvcResult result = mockMvc.perform(put(META_PATH + "/Content-Type")
+ .contentType("application/msword")
+ .accept(MediaType.TEXT_PLAIN_VALUE)
+ .content(partialContent))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String content = result.getResponse().getContentAsString();
+ assertNotNull(content);
+ assertTrue(content.length() > 0, "Should return content type value");
+ }
+
+ @Test
+ public void testGetField_ContentType_JSON_Partial_Found() throws Exception {
+ InputStream docStream = getClass().getResourceAsStream(TEST_DOC);
+ byte[] partialContent = copyBytes(docStream, 12000);
+
+ MvcResult result = mockMvc.perform(put(META_PATH + "/Content-Type")
+ .contentType("application/msword")
+ .accept(MediaType.APPLICATION_JSON_VALUE)
+ .content(partialContent))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String jsonContent = result.getResponse().getContentAsString();
+ Reader reader = new InputStreamReader(new ByteArrayInputStream(jsonContent.getBytes(UTF_8)), UTF_8);
+
+ Metadata metadata = JsonMetadata.fromJson(reader);
+ assertNotNull(metadata.get("Content-Type"));
+ assertEquals(1, metadata.names().length, "Should only return the requested field");
+ }
+
+ /**
+ * Helper method to copy a specified number of bytes from an InputStream.
+ * This simulates partial document uploads for testing partial parsing scenarios.
+ */
+ private byte[] copyBytes(InputStream stream, int maxBytes) throws Exception {
+ byte[] buffer = new byte[maxBytes];
+ int bytesRead = stream.read(buffer);
+ if (bytesRead < maxBytes) {
+ byte[] result = new byte[bytesRead];
+ System.arraycopy(buffer, 0, result, 0, bytesRead);
+ return result;
+ }
+ return buffer;
}
}
diff --git a/tika-server/tika-server-spring/src/test/resources/test-documents/english.txt b/tika-server/tika-server-spring/src/test/resources/test-documents/english.txt
new file mode 100644
index 000000000..dca6086c9
--- /dev/null
+++ b/tika-server/tika-server-spring/src/test/resources/test-documents/english.txt
@@ -0,0 +1,2 @@
+This is a sample English text for language detection testing. The quick brown fox jumps over the lazy dog. Language detection algorithms should be able to identify this text as English based on common words and patterns. This text contains enough content to provide a reliable language detection result.
+
diff --git a/tika-server/tika-server-spring/src/test/resources/test-documents/french.txt b/tika-server/tika-server-spring/src/test/resources/test-documents/french.txt
new file mode 100644
index 000000000..ae68dae23
--- /dev/null
+++ b/tika-server/tika-server-spring/src/test/resources/test-documents/french.txt
@@ -0,0 +1,2 @@
+Ceci est un exemple de texte français pour tester la détection de langue. Le renard brun et rapide saute par-dessus le chien paresseux. Les algorithmes de détection de langue devraient être capables d'identifier ce texte comme étant du français basé sur les mots communs et les modèles. Ce texte contient suffisamment de contenu pour fournir un résultat fiable de détection de langue.
+
diff --git a/tika-server/tika-server-spring/src/test/resources/test-documents/password-protected.doc b/tika-server/tika-server-spring/src/test/resources/test-documents/password-protected.doc
new file mode 100644
index 000000000..b407783d2
Binary files /dev/null and b/tika-server/tika-server-spring/src/test/resources/test-documents/password-protected.doc differ
diff --git a/tika-server/tika-server-spring/src/test/resources/test-documents/test.doc b/tika-server/tika-server-spring/src/test/resources/test-documents/test.doc
new file mode 100644
index 000000000..93198c87c
Binary files /dev/null and b/tika-server/tika-server-spring/src/test/resources/test-documents/test.doc differ
(tika) 03/04: TIKA-4334: replace pipes and server with newer apis - add eval and detector controllers
Posted by nd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ndipiazza pushed a commit to branch TIKA-4334
in repository https://gitbox.apache.org/repos/asf/tika.git
commit e46345a43000a399ecf22588c7cccc3fa32c01a3
Author: Nicholas DiPiazza <nd...@apache.org>
AuthorDate: Tue Sep 30 12:31:09 2025 -0500
TIKA-4334: replace pipes and server with newer apis - add eval and detector controllers
---
tika-server/tika-server-spring/pom.xml | 13 +-
.../tika/server/config/TikaConfigLoader.java | 53 +++
.../tika/server/controller/DetectorController.java | 42 ++-
.../tika/server/controller/EvalController.java | 215 ++++++++++-
.../org/apache/tika/server/util/TikaResource.java | 119 ------
.../main/resources/api/tika-server-openapi.yaml | 43 ++-
.../apache/tika/server/IntegrationTestBase.java | 4 +-
.../DetectorControllerIntegrationTest.java | 42 ++-
.../controller/EvalControllerIntegrationTest.java | 403 +++++++++++++++++++++
.../src/test/resources/test-tika-config.xml | 38 ++
10 files changed, 801 insertions(+), 171 deletions(-)
diff --git a/tika-server/tika-server-spring/pom.xml b/tika-server/tika-server-spring/pom.xml
index 9db1c1884..5fbd2c67b 100644
--- a/tika-server/tika-server-spring/pom.xml
+++ b/tika-server/tika-server-spring/pom.xml
@@ -29,7 +29,7 @@
</parent>
<artifactId>tika-server-spring</artifactId>
- <name>Apache Tika Spring Boot Server</name>
+ <name>Apache Tika Spring Web Service</name>
<description>Apache Tika Server implemented with Spring Boot and OpenAPI 3.0</description>
<dependencyManagement>
@@ -94,6 +94,17 @@
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-eval-core</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-langdetect-opennlp</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.tika</groupId>
diff --git a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/config/TikaConfigLoader.java b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/config/TikaConfigLoader.java
new file mode 100644
index 000000000..d388855ff
--- /dev/null
+++ b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/config/TikaConfigLoader.java
@@ -0,0 +1,53 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ *
+ */
+
+package org.apache.tika.server.config;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.env.Environment;
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
+
+@Configuration
+public class TikaConfigLoader {
+ private final Environment environment;
+
+ @Autowired
+ public TikaConfigLoader(Environment environment) {
+ this.environment = environment;
+ }
+
+ @Bean
+ public TikaConfig tikaConfig() throws TikaException {
+ String tikaConfig = environment.getProperty("tika.config");
+ if (StringUtils.isNotBlank(tikaConfig)) {
+ try {
+ return new TikaConfig(getClass().getClassLoader().getResourceAsStream(tikaConfig));
+ } catch (Exception e) {
+ throw new TikaException("Could not load tika.config profile", e);
+ }
+ }
+ return TikaConfig.getDefaultConfig();
+ }
+}
diff --git a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/DetectorController.java b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/DetectorController.java
index dd1422de6..b4f62d083 100644
--- a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/DetectorController.java
+++ b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/DetectorController.java
@@ -18,23 +18,25 @@ package org.apache.tika.server.controller;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Optional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.context.request.NativeWebRequest;
+import org.apache.tika.config.TikaConfig;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
-import org.apache.tika.parser.ParseContext;
import org.apache.tika.server.api.DetectorResourceApi;
import org.apache.tika.server.component.ServerStatus;
-import org.apache.tika.server.util.TikaResource;
/**
* Controller for MIME/media type detection using the default detector.
@@ -42,45 +44,51 @@ import org.apache.tika.server.util.TikaResource;
*/
@RestController
public class DetectorController implements DetectorResourceApi {
-
private static final Logger LOG = LoggerFactory.getLogger(DetectorController.class);
private final ServerStatus serverStatus;
+ private final TikaConfig tikaConfig;
+
+ @Value("${tika.detector.taskTimeoutMillis:30000}")
+ private long timeoutMillis;
+
@Autowired
- public DetectorController(ServerStatus serverStatus) {
+ public DetectorController(ServerStatus serverStatus, TikaConfig tikaConfig) {
this.serverStatus = serverStatus;
+ this.tikaConfig = tikaConfig;
+ }
+
+ @Override
+ public Optional<NativeWebRequest> getRequest() {
+ return DetectorResourceApi.super.getRequest();
}
@Override
- public ResponseEntity<String> putStream(Resource body) {
+ public ResponseEntity<String> putStream(Resource body, String contentDisposition) {
if (body == null) {
return ResponseEntity.badRequest().body("No document provided");
}
-
+
Metadata metadata = new Metadata();
String filename = body.getFilename();
LOG.info("Detecting media type for Filename: {}", filename);
-
+
if (filename != null) {
metadata.add(TikaCoreProperties.RESOURCE_NAME_KEY, filename);
}
-
- ParseContext parseContext = new ParseContext();
- long timeoutMillis = TikaResource.getTaskTimeout(parseContext);
+
long taskId = serverStatus.start(ServerStatus.TASK.DETECT, filename, timeoutMillis);
try (InputStream is = body.getInputStream();
TikaInputStream tis = TikaInputStream.get(is)) {
-
- String mediaType = TikaResource
- .getConfig()
+
+ String mediaType = tikaConfig
.getDetector()
.detect(tis, metadata)
.toString();
-
+
LOG.info("Detected media type: {} for file: {}", mediaType, filename);
return ResponseEntity.ok(mediaType);
-
} catch (IOException e) {
LOG.warn("Unable to detect MIME type for file. Reason: {} ({})", e.getMessage(), filename, e);
return ResponseEntity.ok(MediaType.OCTET_STREAM.toString());
@@ -88,11 +96,11 @@ public class DetectorController implements DetectorResourceApi {
LOG.error("OOM while detecting: ({})", filename, e);
serverStatus.setStatus(ServerStatus.STATUS.ERROR);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
- .body("Out of memory error during detection");
+ .body("Out of memory error during detection");
} catch (Throwable e) {
LOG.error("Exception while detecting: ({})", filename, e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
- .body("Error during MIME type detection: " + e.getMessage());
+ .body("Error during MIME type detection: " + e.getMessage());
} finally {
serverStatus.complete(taskId);
}
diff --git a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/EvalController.java b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/EvalController.java
index 0e39c28a0..cc0ed5238 100644
--- a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/EvalController.java
+++ b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/controller/EvalController.java
@@ -20,30 +20,231 @@
package org.apache.tika.server.controller;
-import java.util.Optional;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
-import org.springframework.web.context.request.NativeWebRequest;
+import org.springframework.stereotype.Controller;
+import org.apache.tika.eval.core.langid.LanguageIDWrapper;
+import org.apache.tika.eval.core.metadata.TikaEvalMetadataFilter;
+import org.apache.tika.eval.core.textstats.BasicTokenCountStatsCalculator;
+import org.apache.tika.eval.core.textstats.CommonTokens;
+import org.apache.tika.eval.core.textstats.CompositeTextStatsCalculator;
+import org.apache.tika.eval.core.textstats.TextStatsCalculator;
+import org.apache.tika.eval.core.tokens.CommonTokenResult;
+import org.apache.tika.eval.core.tokens.ContrastStatistics;
+import org.apache.tika.eval.core.tokens.TokenContraster;
+import org.apache.tika.eval.core.tokens.TokenCounts;
+import org.apache.tika.language.detect.LanguageResult;
+import org.apache.tika.metadata.Property;
import org.apache.tika.server.api.EvalResourceApi;
+import org.apache.tika.server.component.ServerStatus;
import org.apache.tika.server.model.PutEvalCompare200Response;
import org.apache.tika.server.model.PutEvalCompareRequest;
import org.apache.tika.server.model.PutEvalProfile200Response;
import org.apache.tika.server.model.PutEvalProfileRequest;
+import org.apache.tika.utils.StringUtils;
+@Controller
public class EvalController implements EvalResourceApi {
- @Override
- public Optional<NativeWebRequest> getRequest() {
- return EvalResourceApi.super.getRequest();
+
+ public static final long DEFAULT_TIMEOUT_MILLIS = 60000;
+
+ public static final Property DICE = Property.externalReal(
+ TikaEvalMetadataFilter.TIKA_EVAL_NS + "dice");
+
+ public static final Property OVERLAP = Property.externalReal(
+ TikaEvalMetadataFilter.TIKA_EVAL_NS + "overlap");
+
+ static CompositeTextStatsCalculator TEXT_STATS_CALCULATOR;
+
+ static {
+ List<TextStatsCalculator> calcs = new ArrayList<>();
+ calcs.add(new BasicTokenCountStatsCalculator());
+ calcs.add(new CommonTokens());
+ TEXT_STATS_CALCULATOR = new CompositeTextStatsCalculator(calcs);
}
+ @Autowired
+ private ServerStatus serverStatus;
+
@Override
public ResponseEntity<PutEvalCompare200Response> putEvalCompare(PutEvalCompareRequest putEvalCompareRequest) {
- return EvalResourceApi.super.putEvalCompare(putEvalCompareRequest);
+ try {
+ String id = putEvalCompareRequest.getId();
+ String textA = putEvalCompareRequest.getTextA();
+ String textB = putEvalCompareRequest.getTextB();
+ long timeoutMillis = putEvalCompareRequest.getTimeoutMillis() != null ?
+ putEvalCompareRequest.getTimeoutMillis() : DEFAULT_TIMEOUT_MILLIS;
+
+ Map<String, Object> result = compareText(id, textA, textB, timeoutMillis);
+
+ PutEvalCompare200Response response = new PutEvalCompare200Response();
+ mapResultToCompareResponse(result, response);
+
+ return new ResponseEntity<>(response, HttpStatus.OK);
+ } catch (Exception e) {
+ return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
+ }
}
@Override
public ResponseEntity<PutEvalProfile200Response> putEvalProfile(PutEvalProfileRequest putEvalProfileRequest) {
- return EvalResourceApi.super.putEvalProfile(putEvalProfileRequest);
+ try {
+ String id = putEvalProfileRequest.getId();
+ String text = putEvalProfileRequest.getText();
+ long timeoutMillis = putEvalProfileRequest.getTimeoutMillis() != null ?
+ putEvalProfileRequest.getTimeoutMillis() : DEFAULT_TIMEOUT_MILLIS;
+
+ Map<String, Object> result = profile(id, text, timeoutMillis);
+
+ PutEvalProfile200Response response = new PutEvalProfile200Response();
+ mapResultToProfileResponse(result, response);
+
+ return new ResponseEntity<>(response, HttpStatus.OK);
+ } catch (Exception e) {
+ return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ private Map<String, Object> compareText(String id, String textA, String textB, long timeoutMillis) {
+ Map<String, Object> stats = new HashMap<>();
+ long taskId = serverStatus.start(ServerStatus.TASK.EVAL, id, timeoutMillis);
+ try {
+ TokenCounts tokensA = profile("A", textA, stats);
+ TokenCounts tokensB = profile("B", textB, stats);
+ TokenContraster tokenContraster = new TokenContraster();
+ ContrastStatistics contrastStatistics =
+ tokenContraster.calculateContrastStatistics(tokensA, tokensB);
+ reportContrastStats(contrastStatistics, stats);
+ } finally {
+ serverStatus.complete(taskId);
+ }
+ return stats;
+ }
+
+ private Map<String, Object> profile(String id, String text, long timeoutMillis) {
+ Map<String, Object> stats = new HashMap<>();
+ long taskId = serverStatus.start(ServerStatus.TASK.EVAL, id, timeoutMillis);
+ try {
+ profile(StringUtils.EMPTY, text, stats);
+ } finally {
+ serverStatus.complete(taskId);
+ }
+ return stats;
+ }
+
+ private TokenCounts profile(String suffix, String content, Map<String, Object> stats) {
+ Map<Class, Object> results = TEXT_STATS_CALCULATOR.calculate(content);
+
+ TokenCounts tokenCounts = (TokenCounts) results.get(BasicTokenCountStatsCalculator.class);
+ stats.put("tika-eval:numTokens" + suffix, tokenCounts.getTotalTokens());
+ stats.put("tika-eval:numUniqueTokens" + suffix, tokenCounts.getTotalUniqueTokens());
+
+ //common token results
+ CommonTokenResult commonTokenResult = (CommonTokenResult) results.get(CommonTokens.class);
+ stats.put("tika-eval:numAlphaTokens" + suffix, commonTokenResult.getAlphabeticTokens());
+ stats.put("tika-eval:numUniqueAlphaTokens" + suffix, commonTokenResult.getUniqueAlphabeticTokens());
+ if (commonTokenResult.getAlphabeticTokens() > 0) {
+ stats.put("tika-eval:oov" + suffix, commonTokenResult.getOOV());
+ } else {
+ stats.put("tika-eval:oov" + suffix, -1.0f);
+ }
+
+ //languages
+ List<LanguageResult> probabilities =
+ (List<LanguageResult>) results.get(LanguageIDWrapper.class);
+ if (probabilities.size() > 0) {
+ stats.put("tika-eval:lang" + suffix, probabilities.get(0).getLanguage());
+ stats.put("tika-eval:langConfidence" + suffix, probabilities.get(0).getRawScore());
+ }
+ return tokenCounts;
+ }
+
+ private void reportContrastStats(ContrastStatistics contrastStatistics,
+ Map<String, Object> stats) {
+ stats.put("tika-eval:dice", contrastStatistics.getDiceCoefficient());
+ stats.put("tika-eval:overlap", contrastStatistics.getOverlap());
+ //TODO, add topNMore, topNUnique
+ }
+
+ private void mapResultToCompareResponse(Map<String, Object> result, PutEvalCompare200Response response) {
+ if (result.get("tika-eval:dice") != null) {
+ response.setTikaEvalColonDice(((Number) result.get("tika-eval:dice")).floatValue());
+ }
+ if (result.get("tika-eval:overlap") != null) {
+ response.setTikaEvalColonOverlap(((Number) result.get("tika-eval:overlap")).floatValue());
+ }
+ if (result.get("tika-eval:numTokensA") != null) {
+ response.setTikaEvalColonNumTokensA(((Number) result.get("tika-eval:numTokensA")).intValue());
+ }
+ if (result.get("tika-eval:numTokensB") != null) {
+ response.setTikaEvalColonNumTokensB(((Number) result.get("tika-eval:numTokensB")).intValue());
+ }
+ if (result.get("tika-eval:numUniqueTokensA") != null) {
+ response.setTikaEvalColonNumUniqueTokensA(((Number) result.get("tika-eval:numUniqueTokensA")).intValue());
+ }
+ if (result.get("tika-eval:numUniqueTokensB") != null) {
+ response.setTikaEvalColonNumUniqueTokensB(((Number) result.get("tika-eval:numUniqueTokensB")).intValue());
+ }
+ if (result.get("tika-eval:numAlphaTokensA") != null) {
+ response.setTikaEvalColonNumAlphaTokensA(((Number) result.get("tika-eval:numAlphaTokensA")).intValue());
+ }
+ if (result.get("tika-eval:numAlphaTokensB") != null) {
+ response.setTikaEvalColonNumAlphaTokensB(((Number) result.get("tika-eval:numAlphaTokensB")).intValue());
+ }
+ if (result.get("tika-eval:numUniqueAlphaTokensA") != null) {
+ response.setTikaEvalColonNumUniqueAlphaTokensA(((Number) result.get("tika-eval:numUniqueAlphaTokensA")).intValue());
+ }
+ if (result.get("tika-eval:numUniqueAlphaTokensB") != null) {
+ response.setTikaEvalColonNumUniqueAlphaTokensB(((Number) result.get("tika-eval:numUniqueAlphaTokensB")).intValue());
+ }
+ if (result.get("tika-eval:oovA") != null) {
+ response.setTikaEvalColonOovA(((Number) result.get("tika-eval:oovA")).floatValue());
+ }
+ if (result.get("tika-eval:oovB") != null) {
+ response.setTikaEvalColonOovB(((Number) result.get("tika-eval:oovB")).floatValue());
+ }
+ if (result.get("tika-eval:langA") != null) {
+ response.setTikaEvalColonLangA((String) result.get("tika-eval:langA"));
+ }
+ if (result.get("tika-eval:langB") != null) {
+ response.setTikaEvalColonLangB((String) result.get("tika-eval:langB"));
+ }
+ if (result.get("tika-eval:langConfidenceA") != null) {
+ response.setTikaEvalColonLangConfidenceA(((Number) result.get("tika-eval:langConfidenceA")).floatValue());
+ }
+ if (result.get("tika-eval:langConfidenceB") != null) {
+ response.setTikaEvalColonLangConfidenceB(((Number) result.get("tika-eval:langConfidenceB")).floatValue());
+ }
+ }
+
+ private void mapResultToProfileResponse(Map<String, Object> result, PutEvalProfile200Response response) {
+ if (result.get("tika-eval:numTokens") != null) {
+ response.setTikaEvalColonNumTokens(((Number) result.get("tika-eval:numTokens")).intValue());
+ }
+ if (result.get("tika-eval:numUniqueTokens") != null) {
+ response.setTikaEvalColonNumUniqueTokens(((Number) result.get("tika-eval:numUniqueTokens")).intValue());
+ }
+ if (result.get("tika-eval:numAlphaTokens") != null) {
+ response.setTikaEvalColonNumAlphaTokens(((Number) result.get("tika-eval:numAlphaTokens")).intValue());
+ }
+ if (result.get("tika-eval:numUniqueAlphaTokens") != null) {
+ response.setTikaEvalColonNumUniqueAlphaTokens(((Number) result.get("tika-eval:numUniqueAlphaTokens")).intValue());
+ }
+ if (result.get("tika-eval:oov") != null) {
+ response.setTikaEvalColonOov(((Number) result.get("tika-eval:oov")).floatValue());
+ }
+ if (result.get("tika-eval:lang") != null) {
+ response.setTikaEvalColonLang((String) result.get("tika-eval:lang"));
+ }
+ if (result.get("tika-eval:langConfidence") != null) {
+ response.setTikaEvalColonLangConfidence(((Number) result.get("tika-eval:langConfidence")).floatValue());
+ }
}
}
diff --git a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/util/TikaResource.java b/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/util/TikaResource.java
deleted file mode 100644
index d810b14bb..000000000
--- a/tika-server/tika-server-spring/src/main/java/org/apache/tika/server/util/TikaResource.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- *
- */
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.util;
-
-import org.springframework.stereotype.Component;
-
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.detect.Detector;
-import org.apache.tika.parser.ParseContext;
-
-/**
- * Utility class providing access to Tika configuration and common operations.
- * This class serves as a bridge between the Spring controllers and Tika core functionality.
- */
-@Component
-public class TikaResource {
-
- private static final long DEFAULT_TASK_TIMEOUT_MILLIS = 300000; // 5 minutes
- private static TikaConfig tikaConfig;
-
- static {
- try {
- // Initialize with default Tika configuration
- tikaConfig = TikaConfig.getDefaultConfig();
- } catch (Exception e) {
- throw new RuntimeException("Failed to initialize TikaConfig", e);
- }
- }
-
- /**
- * Get the task timeout from ParseContext, or return default if not configured.
- *
- * @param parseContext The parse context which may contain timeout configuration
- * @return Timeout in milliseconds
- */
- public static long getTaskTimeout(ParseContext parseContext) {
- // Check if timeout is configured in parse context
- if (parseContext != null) {
- // Look for timeout configuration - this could be expanded based on actual Tika implementation
- Object timeout = parseContext.get(Object.class); // Placeholder - actual implementation would vary
- if (timeout instanceof Long) {
- return (Long) timeout;
- }
- }
- return DEFAULT_TASK_TIMEOUT_MILLIS;
- }
-
- /**
- * Get the Tika configuration instance.
- *
- * @return TikaConfig instance
- */
- public static TikaConfig getConfig() {
- return tikaConfig;
- }
-
- /**
- * Get the detector from the Tika configuration.
- *
- * @return Detector instance
- */
- public static Detector getDetector() {
- return tikaConfig.getDetector();
- }
-
- /**
- * Set a custom TikaConfig (useful for testing or custom configurations).
- *
- * @param config The TikaConfig to use
- */
- public static void setConfig(TikaConfig config) {
- tikaConfig = config;
- }
-
- /**
- * Reset to default configuration.
- */
- public static void resetToDefault() {
- try {
- tikaConfig = TikaConfig.getDefaultConfig();
- } catch (Exception e) {
- throw new RuntimeException("Failed to reset to default TikaConfig", e);
- }
- }
-}
diff --git a/tika-server/tika-server-spring/src/main/resources/api/tika-server-openapi.yaml b/tika-server/tika-server-spring/src/main/resources/api/tika-server-openapi.yaml
index 6a2d738b4..eee596a91 100644
--- a/tika-server/tika-server-spring/src/main/resources/api/tika-server-openapi.yaml
+++ b/tika-server/tika-server-spring/src/main/resources/api/tika-server-openapi.yaml
@@ -177,11 +177,20 @@ paths:
operationId: put_stream
requestBody:
content:
- '*/*':
+ application/octet-stream:
schema:
type: string
format: binary
required: true
+ parameters:
+ - in: header
+ name: Content-Disposition
+ description: |
+ Optional header to provide a filename hint to Tika for more accurate detection.
+ Example: `attachment; filename="document.pdf"`
+ schema:
+ type: string
+ required: false
responses:
'200':
content:
@@ -1284,12 +1293,34 @@ paths:
"tika-eval:numAlphaTokensB":
type: integer
description: Number of alphabetic tokens in text B
- "tika-eval:languageA":
+ "tika-eval:numUniqueAlphaTokensA":
+ type: integer
+ description: Number of unique alphabetic tokens in text A
+ "tika-eval:numUniqueAlphaTokensB":
+ type: integer
+ description: Number of unique alphabetic tokens in text B
+ "tika-eval:oovA":
+ type: number
+ format: float
+ description: Out-of-vocabulary ratio for text A
+ "tika-eval:oovB":
+ type: number
+ format: float
+ description: Out-of-vocabulary ratio for text B
+ "tika-eval:langA":
type: string
description: Detected language for text A
- "tika-eval:languageB":
+ "tika-eval:langB":
type: string
description: Detected language for text B
+ "tika-eval:langConfidenceA":
+ type: number
+ format: float
+ description: Language detection confidence for text A
+ "tika-eval:langConfidenceB":
+ type: number
+ format: float
+ description: Language detection confidence for text B
description: If successful, this operation returns HTTP status code 200 with comparison statistics and similarity metrics.
'500':
description: An error occurred processing the call.
@@ -1342,15 +1373,15 @@ paths:
"tika-eval:numUniqueAlphaTokens":
type: integer
description: Number of unique alphabetic tokens
- "tika-eval:outOfVocabulary":
+ "tika-eval:oov":
type: number
format: float
description: Out-of-vocabulary ratio (-1.0 if no alphabetic tokens)
- "tika-eval:language":
+ "tika-eval:lang":
type: string
description: Detected language code
example: "en"
- "tika-eval:languageConfidence":
+ "tika-eval:langConfidence":
type: number
format: float
description: Language detection confidence score
diff --git a/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/IntegrationTestBase.java b/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/IntegrationTestBase.java
index e8ebfc607..c030e97eb 100644
--- a/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/IntegrationTestBase.java
+++ b/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/IntegrationTestBase.java
@@ -29,7 +29,7 @@ import org.springframework.web.context.WebApplicationContext;
* and setup for Spring Boot tests with a random port.
*/
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@TestPropertySource(properties = {"tika.config.path=classpath:test-tika-config.xml"})
+@TestPropertySource(properties = {"tika.config=test-tika-config.xml"})
public abstract class IntegrationTestBase {
@LocalServerPort
@@ -45,4 +45,4 @@ public abstract class IntegrationTestBase {
protected String getBaseUrl() {
return "http://localhost:" + port;
}
-}
\ No newline at end of file
+}
diff --git a/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/DetectorControllerIntegrationTest.java b/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/DetectorControllerIntegrationTest.java
index 3a2cca8e9..015c732e9 100644
--- a/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/DetectorControllerIntegrationTest.java
+++ b/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/DetectorControllerIntegrationTest.java
@@ -27,8 +27,6 @@ import java.nio.file.Paths;
import org.junit.jupiter.api.Test;
import org.springframework.http.MediaType;
-import org.springframework.mock.web.MockMultipartFile;
-import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.apache.tika.server.IntegrationTestBase;
@@ -55,7 +53,8 @@ public class DetectorControllerIntegrationTest extends IntegrationTestBase {
mockMvc.perform(put("/detect/stream")
.contentType(MediaType.APPLICATION_OCTET_STREAM)
- .content(jsonContent))
+ .content(jsonContent)
+ .header("Content-Disposition", "attachment; filename=\"test.json\""))
.andExpect(status().isOk())
.andExpect(content().string("application/json"));
}
@@ -101,11 +100,12 @@ public class DetectorControllerIntegrationTest extends IntegrationTestBase {
public void testDetectEmptyContent() throws Exception {
byte[] emptyContent = new byte[0];
+ // Empty content is treated as missing request body by Spring, so expect 400
mockMvc.perform(put("/detect/stream")
.contentType(MediaType.APPLICATION_OCTET_STREAM)
- .content(emptyContent))
- .andExpect(status().isOk())
- .andExpect(content().string("application/octet-stream"));
+ .content(emptyContent)
+ .header("Content-Disposition", "attachment; filename=\"empty.txt\""))
+ .andExpect(status().isBadRequest());
}
@Test
@@ -126,9 +126,10 @@ public class DetectorControllerIntegrationTest extends IntegrationTestBase {
byte[] textContent = "This is a test file.".getBytes(StandardCharsets.UTF_8);
// Test that filename hints help with detection
- mockMvc.perform(MockMvcRequestBuilders.multipart("/detect/stream")
- .file(new MockMultipartFile("file", "test.txt",
- MediaType.TEXT_PLAIN_VALUE, textContent)))
+ mockMvc.perform(put("/detect/stream")
+ .contentType(MediaType.APPLICATION_OCTET_STREAM)
+ .content(textContent)
+ .header("Content-Disposition", "attachment; filename=\"test.txt\""))
.andExpect(status().isOk())
.andExpect(content().string("text/plain"));
}
@@ -139,9 +140,10 @@ public class DetectorControllerIntegrationTest extends IntegrationTestBase {
mockMvc.perform(put("/detect/stream")
.contentType(MediaType.APPLICATION_OCTET_STREAM)
- .content(jsContent))
+ .content(jsContent)
+ .header("Content-Disposition", "attachment; filename=\"test.js\""))
.andExpect(status().isOk())
- .andExpect(content().string("application/javascript"));
+ .andExpect(content().string("text/javascript"));
}
@Test
@@ -150,7 +152,8 @@ public class DetectorControllerIntegrationTest extends IntegrationTestBase {
mockMvc.perform(put("/detect/stream")
.contentType(MediaType.APPLICATION_OCTET_STREAM)
- .content(cssContent))
+ .content(cssContent)
+ .header("Content-Disposition", "attachment; filename=\"style.css\""))
.andExpect(status().isOk())
.andExpect(content().string("text/css"));
}
@@ -175,8 +178,7 @@ public class DetectorControllerIntegrationTest extends IntegrationTestBase {
public void testDetectNullContent() throws Exception {
// Test with no content - should return bad request
mockMvc.perform(put("/detect/stream"))
- .andExpect(status().isBadRequest())
- .andExpect(content().string("No document provided"));
+ .andExpect(status().isBadRequest());
}
@Test
@@ -238,10 +240,10 @@ public class DetectorControllerIntegrationTest extends IntegrationTestBase {
mockMvc.perform(put("/detect/stream")
.contentType(MediaType.APPLICATION_OCTET_STREAM)
- .content(markdownContent))
+ .content(markdownContent)
+ .header("Content-Disposition", "attachment; filename=\"test.md\""))
.andExpect(status().isOk())
- // Markdown might be detected as text/plain since it's text-based
- .andExpect(content().string("text/plain"));
+ .andExpect(content().string("text/x-web-markdown"));
}
@Test
@@ -251,7 +253,8 @@ public class DetectorControllerIntegrationTest extends IntegrationTestBase {
mockMvc.perform(put("/detect/stream")
.contentType(MediaType.APPLICATION_OCTET_STREAM)
- .content(csvContent))
+ .content(csvContent)
+ .header("Content-Disposition", "attachment; filename=\"test.csv\""))
.andExpect(status().isOk())
.andExpect(content().string("text/csv"));
}
@@ -311,7 +314,8 @@ public class DetectorControllerIntegrationTest extends IntegrationTestBase {
mockMvc.perform(put("/detect/stream")
.contentType(MediaType.APPLICATION_OCTET_STREAM)
- .content(jsonContent))
+ .content(jsonContent)
+ .header("Content-Disposition", "attachment; filename=\"test.json\""))
.andExpect(status().isOk())
.andExpect(content().string("application/json"));
}
diff --git a/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/EvalControllerIntegrationTest.java b/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/EvalControllerIntegrationTest.java
new file mode 100644
index 000000000..c0720a6f0
--- /dev/null
+++ b/tika-server/tika-server-spring/src/test/java/org/apache/tika/server/controller/EvalControllerIntegrationTest.java
@@ -0,0 +1,403 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.controller;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+import org.springframework.http.MediaType;
+import org.springframework.test.web.servlet.MvcResult;
+
+import org.apache.tika.server.IntegrationTestBase;
+
+/**
+ * Integration tests for EvalController.
+ * Tests text profiling and comparison endpoints using TikaEval framework.
+ */
+public class EvalControllerIntegrationTest extends IntegrationTestBase {
+
+ private final ObjectMapper objectMapper = new ObjectMapper();
+
+ @Test
+ public void testBasicProfile() throws Exception {
+ Map<String, String> request = new HashMap<>();
+ request.put("id", "1");
+ request.put("text", "the quick brown fox jumped qwertyuiop");
+
+ String jsonRequest = objectMapper.writeValueAsString(request);
+
+ MvcResult result = mockMvc.perform(put("/eval/profile")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String responseBody = result.getResponse().getContentAsString();
+ Map<String, Object> results = objectMapper.readValue(responseBody,
+ new TypeReference<Map<String, Object>>() {});
+
+ // Verify token count - based on original test expectations
+ Integer numTokens = (Integer) results.get("tika-eval:numTokens");
+ assertEquals(6, numTokens.intValue());
+
+ Object oovObj = results.get("tika-eval:oov");
+ Double oov = ((Number) oovObj).doubleValue();
+ assertEquals(0.166, oov, 0.01);
+
+ // Verify language detection
+ String language = (String) results.get("tika-eval:lang");
+ assertNotNull(language);
+ }
+
+ @Test
+ public void testBasicCompare() throws Exception {
+ Map<String, String> request = new HashMap<>();
+ request.put("id", "1");
+ request.put("textA", "the quick brown fox jumped qwertyuiop");
+ request.put("textB", "the the the fast brown dog jumped qwertyuiop");
+
+ String jsonRequest = objectMapper.writeValueAsString(request);
+
+ MvcResult result = mockMvc.perform(put("/eval/compare")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String responseBody = result.getResponse().getContentAsString();
+ Map<String, Object> results = objectMapper.readValue(responseBody,
+ new TypeReference<Map<String, Object>>() {});
+
+ // Verify text A statistics - based on original test expectations
+ Integer numTokensA = (Integer) results.get("tika-eval:numTokensA");
+ assertEquals(6, numTokensA.intValue());
+
+ Object oovAObj = results.get("tika-eval:oovA");
+ Double oovA = ((Number) oovAObj).doubleValue();
+ assertEquals(0.166, oovA, 0.01);
+
+ String languageA = (String) results.get("tika-eval:langA");
+ assertNotNull(languageA);
+
+ // Verify similarity metrics - based on original test expectations
+ Object diceObj = results.get("tika-eval:dice");
+ Double dice = ((Number) diceObj).doubleValue();
+ assertEquals(0.666, dice, 0.01);
+
+ Object overlapObj = results.get("tika-eval:overlap");
+ Double overlap = ((Number) overlapObj).doubleValue();
+ assertEquals(0.571, overlap, 0.01);
+ }
+
+ @Test
+ public void testProfileWithTimeout() throws Exception {
+ Map<String, Object> request = new HashMap<>();
+ request.put("id", "timeout-test");
+ request.put("text", "short text for testing timeout functionality");
+ request.put("timeoutMillis", 30000);
+
+ String jsonRequest = objectMapper.writeValueAsString(request);
+
+ MvcResult result = mockMvc.perform(put("/eval/profile")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String responseBody = result.getResponse().getContentAsString();
+ Map<String, Object> results = objectMapper.readValue(responseBody,
+ new TypeReference<Map<String, Object>>() {});
+
+ // Verify basic statistics for the text
+ assertNotNull(results.get("tika-eval:numTokens"));
+ assertNotNull(results.get("tika-eval:numUniqueTokens"));
+ }
+
+ @Test
+ public void testCompareWithTimeout() throws Exception {
+ Map<String, Object> request = new HashMap<>();
+ request.put("id", "compare-timeout-test");
+ request.put("textA", "hello world");
+ request.put("textB", "hello universe");
+ request.put("timeoutMillis", 30000);
+
+ String jsonRequest = objectMapper.writeValueAsString(request);
+
+ MvcResult result = mockMvc.perform(put("/eval/compare")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String responseBody = result.getResponse().getContentAsString();
+ Map<String, Object> results = objectMapper.readValue(responseBody,
+ new TypeReference<Map<String, Object>>() {});
+
+ // Verify both texts have been processed
+ assertNotNull(results.get("tika-eval:numTokensA"));
+ assertNotNull(results.get("tika-eval:numTokensB"));
+
+ // Verify similarity metrics are present
+ assertNotNull(results.get("tika-eval:dice"));
+ assertNotNull(results.get("tika-eval:overlap"));
+ }
+
+ @Test
+ public void testProfileEmptyText() throws Exception {
+ Map<String, String> request = new HashMap<>();
+ request.put("id", "empty-test");
+ request.put("text", "");
+
+ String jsonRequest = objectMapper.writeValueAsString(request);
+
+ MvcResult result = mockMvc.perform(put("/eval/profile")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String responseBody = result.getResponse().getContentAsString();
+ Map<String, Object> results = objectMapper.readValue(responseBody,
+ new TypeReference<Map<String, Object>>() {});
+
+ // Empty text should have zero tokens
+ assertEquals(0, (Integer) results.get("tika-eval:numTokens"));
+ assertEquals(0, (Integer) results.get("tika-eval:numUniqueTokens"));
+ assertEquals(0, (Integer) results.get("tika-eval:numAlphaTokens"));
+ }
+
+ @Test
+ public void testProfileLongText() throws Exception {
+ // Create a longer text sample for testing
+ StringBuilder longText = new StringBuilder();
+ for (int i = 0; i < 100; i++) {
+ longText.append("This is sentence number ").append(i).append(". ");
+ }
+
+ Map<String, String> request = new HashMap<>();
+ request.put("id", "long-text-test");
+ request.put("text", longText.toString());
+
+ String jsonRequest = objectMapper.writeValueAsString(request);
+
+ MvcResult result = mockMvc.perform(put("/eval/profile")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String responseBody = result.getResponse().getContentAsString();
+ Map<String, Object> results = objectMapper.readValue(responseBody,
+ new TypeReference<Map<String, Object>>() {});
+
+ // Verify we get reasonable statistics for long text
+ Integer numTokens = (Integer) results.get("tika-eval:numTokens");
+ assertNotNull(numTokens);
+ // Should have many tokens for this long text
+ assert(numTokens > 100);
+
+ assertNotNull(results.get("tika-eval:numUniqueTokens"));
+ }
+
+ @Test
+ public void testCompareIdenticalTexts() throws Exception {
+ Map<String, String> request = new HashMap<>();
+ request.put("id", "identical-test");
+ request.put("textA", "the quick brown fox");
+ request.put("textB", "the quick brown fox");
+
+ String jsonRequest = objectMapper.writeValueAsString(request);
+
+ MvcResult result = mockMvc.perform(put("/eval/compare")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String responseBody = result.getResponse().getContentAsString();
+ Map<String, Object> results = objectMapper.readValue(responseBody,
+ new TypeReference<Map<String, Object>>() {});
+
+ // Identical texts should have high similarity scores
+ Double dice = (Double) results.get("tika-eval:dice");
+ assertNotNull(dice);
+ // Dice coefficient should be close to 1.0 for identical texts
+ assert(dice > 0.9);
+
+ // Token counts should be identical
+ assertEquals(results.get("tika-eval:numTokensA"), results.get("tika-eval:numTokensB"));
+ }
+
+ @Test
+ public void testCompareCompletelyDifferentTexts() throws Exception {
+ Map<String, String> request = new HashMap<>();
+ request.put("id", "different-test");
+ request.put("textA", "apple banana cherry");
+ request.put("textB", "dog elephant frog");
+
+ String jsonRequest = objectMapper.writeValueAsString(request);
+
+ MvcResult result = mockMvc.perform(put("/eval/compare")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String responseBody = result.getResponse().getContentAsString();
+ Map<String, Object> results = objectMapper.readValue(responseBody,
+ new TypeReference<Map<String, Object>>() {});
+
+ // Completely different texts should have low similarity scores
+ Double dice = (Double) results.get("tika-eval:dice");
+ assertNotNull(dice);
+ // Dice coefficient should be close to 0.0 for completely different texts
+ assert(dice < 0.1);
+
+ Double overlap = (Double) results.get("tika-eval:overlap");
+ assertNotNull(overlap);
+ assert(overlap < 0.1);
+ }
+
+ @Test
+ public void testInvalidJsonRequest() throws Exception {
+ String invalidJson = "{invalid json}";
+
+ mockMvc.perform(put("/eval/profile")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(invalidJson.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isBadRequest());
+ }
+
+ @Test
+ public void testMissingRequiredFields() throws Exception {
+ Map<String, String> request = new HashMap<>();
+ request.put("id", "missing-text");
+ // Missing "text" field
+
+ String jsonRequest = objectMapper.writeValueAsString(request);
+
+ mockMvc.perform(put("/eval/profile")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isBadRequest());
+ }
+
+ @Test
+ public void testMissingRequiredFieldsCompare() throws Exception {
+ Map<String, String> request = new HashMap<>();
+ request.put("id", "missing-text");
+ request.put("textA", "some text");
+ // Missing "textB" field
+
+ String jsonRequest = objectMapper.writeValueAsString(request);
+
+ mockMvc.perform(put("/eval/compare")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isBadRequest());
+ }
+
+ @Test
+ public void testMultipleProfileRequests() throws Exception {
+ // Test that the server can handle multiple concurrent profile requests
+ Map<String, String> request1 = new HashMap<>();
+ request1.put("id", "multi-test-1");
+ request1.put("text", "First test text for profiling.");
+
+ Map<String, String> request2 = new HashMap<>();
+ request2.put("id", "multi-test-2");
+ request2.put("text", "Second test text for profiling analysis.");
+
+ Map<String, String> request3 = new HashMap<>();
+ request3.put("id", "multi-test-3");
+ request3.put("text", "Third test text with different content for evaluation.");
+
+ String jsonRequest1 = objectMapper.writeValueAsString(request1);
+ String jsonRequest2 = objectMapper.writeValueAsString(request2);
+ String jsonRequest3 = objectMapper.writeValueAsString(request3);
+
+ // Execute multiple requests to test server status tracking
+ mockMvc.perform(put("/eval/profile")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest1.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isOk());
+
+ mockMvc.perform(put("/eval/profile")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest2.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isOk());
+
+ mockMvc.perform(put("/eval/profile")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest3.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isOk());
+ }
+
+ @Test
+ public void testSpecialCharactersInText() throws Exception {
+ Map<String, String> request = new HashMap<>();
+ request.put("id", "special-chars-test");
+ request.put("text", "Hello, world! This text contains special characters: @#$%^&*()_+{}|:<>?");
+
+ String jsonRequest = objectMapper.writeValueAsString(request);
+
+ MvcResult result = mockMvc.perform(put("/eval/profile")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String responseBody = result.getResponse().getContentAsString();
+ Map<String, Object> results = objectMapper.readValue(responseBody,
+ new TypeReference<Map<String, Object>>() {});
+
+ // Should handle special characters gracefully
+ assertNotNull(results.get("tika-eval:numTokens"));
+ assertNotNull(results.get("tika-eval:numAlphaTokens"));
+ }
+
+ @Test
+ public void testUnicodeText() throws Exception {
+ Map<String, String> request = new HashMap<>();
+ request.put("id", "unicode-test");
+ request.put("text", "Héllo wørld! This is tëst tëxt with ūnïcōdē characters: 你好世界 🌍");
+
+ String jsonRequest = objectMapper.writeValueAsString(request);
+
+ MvcResult result = mockMvc.perform(put("/eval/profile")
+ .contentType(MediaType.APPLICATION_JSON)
+ .content(jsonRequest.getBytes(StandardCharsets.UTF_8)))
+ .andExpect(status().isOk())
+ .andReturn();
+
+ String responseBody = result.getResponse().getContentAsString();
+ Map<String, Object> results = objectMapper.readValue(responseBody,
+ new TypeReference<Map<String, Object>>() {});
+
+ // Should handle Unicode characters gracefully
+ assertNotNull(results.get("tika-eval:numTokens"));
+ }
+}
diff --git a/tika-server/tika-server-spring/src/test/resources/test-tika-config.xml b/tika-server/tika-server-spring/src/test/resources/test-tika-config.xml
new file mode 100644
index 000000000..2e2597ad4
--- /dev/null
+++ b/tika-server/tika-server-spring/src/test/resources/test-tika-config.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <!-- Basic configuration for integration tests -->
+ <parsers>
+ <parser class="org.apache.tika.parser.DefaultParser"/>
+ </parsers>
+
+ <detectors>
+ <detector class="org.apache.tika.detect.DefaultDetector"/>
+ <detector class="org.apache.tika.mime.MimeTypes"/>
+ </detectors>
+
+ <!-- Server-specific configuration -->
+ <service-loader>
+ <dynamic>true</dynamic>
+ </service-loader>
+
+ <!-- Metadata configuration -->
+ <metadataFilter class="org.apache.tika.metadata.filter.MetadataFilter">
+ <!-- Allow all metadata for tests -->
+ </metadataFilter>
+</properties>