You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by nd...@apache.org on 2025/10/01 12:54:10 UTC
(tika) 01/04: TIKA-4334: replace pipes and server with newer apis
This is an automated email from the ASF dual-hosted git repository.
ndipiazza pushed a commit to branch TIKA-4334
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 1c933becbcb06a2a22a323389456aefad43957bf
Author: Nicholas DiPiazza <nd...@apache.org>
AuthorDate: Sun Sep 28 14:25:42 2025 -0500
TIKA-4334: replace pipes and server with newer apis
---
pom.xml | 2 +-
tika-grpc/README.md | 13 -
tika-grpc/pom.xml | 438 ----
.../tika/pipes/grpc/ExpiringFetcherStore.java | 101 -
.../org/apache/tika/pipes/grpc/TikaGrpcServer.java | 205 --
.../apache/tika/pipes/grpc/TikaGrpcServerImpl.java | 420 ----
tika-grpc/src/main/proto/tika.proto | 145 --
.../tika/pipes/grpc/ExpiringFetcherStoreTest.java | 65 -
...PipesBiDirectionalStreamingIntegrationTest.java | 217 --
.../apache/tika/pipes/grpc/TikaGrpcServerTest.java | 287 ---
tika-grpc/src/test/resources/certs/README.md | 5 -
tika-grpc/src/test/resources/certs/ca.key | 28 -
tika-grpc/src/test/resources/certs/ca.pem | 20 -
tika-grpc/src/test/resources/certs/client.key | 28 -
tika-grpc/src/test/resources/certs/client.pem | 20 -
tika-grpc/src/test/resources/certs/server1.key | 28 -
tika-grpc/src/test/resources/certs/server1.pem | 22 -
.../src/test/resources/tika-pipes-test-config.xml | 35 -
tika-parent/pom.xml | 20 +
tika-pipes/pom.xml | 99 +-
tika-pipes/tika-async-cli/pom.xml | 134 --
.../apache/tika/async/cli/SimpleAsyncConfig.java | 60 -
.../org/apache/tika/async/cli/TikaAsyncCLI.java | 159 --
.../tika/async/cli/TikaConfigAsyncWriter.java | 159 --
.../apache/tika/async/cli/AsyncCliParserTest.java | 82 -
.../apache/tika/async/cli/AsyncProcessorTest.java | 145 --
.../apache/tika/async/cli/TikaAsyncCLITest.java | 41 -
.../test/resources/configs/TIKA-4207-emitter.xml | 35 -
.../test/resources/configs/tika-config-broken.xml | 32 -
.../resources/test-documents/basic_embedded.xml | 35 -
tika-pipes/tika-emitters/pom.xml | 47 -
.../tika-emitters/tika-emitter-az-blob/pom.xml | 114 --
.../tika/pipes/emitter/azblob/AZBlobEmitter.java | 265 ---
.../pipes/emitter/azblob/TestAZBlobEmitter.java | 56 -
.../test/resources/config/tika-config-az-blob.xml | 28 -
tika-pipes/tika-emitters/tika-emitter-gcs/pom.xml | 114 --
.../apache/tika/pipes/emitter/gcs/GCSEmitter.java | 195 --
.../tika/pipes/emitter/gcs/TestGCSEmitter.java | 55 -
.../src/test/resources/config/tika-config-gcs.xml | 26 -
tika-pipes/tika-emitters/tika-emitter-jdbc/pom.xml | 59 -
.../tika/pipes/emitter/jdbc/JDBCEmitter.java | 701 -------
.../tika/pipes/emitter/jdbc/JDBCEmitterTest.java | 271 ---
.../tika-config-jdbc-emitter-attachments.xml | 53 -
.../tika-config-jdbc-emitter-existing-table.xml | 42 -
.../tika-config-jdbc-emitter-multivalued.xml | 45 -
.../configs/tika-config-jdbc-emitter-trunc.xml | 44 -
.../resources/configs/tika-config-jdbc-emitter.xml | 54 -
.../tika-emitters/tika-emitter-kafka/pom.xml | 118 --
.../tika/pipes/emitter/kafka/KafkaEmitter.java | 288 ---
.../tika-emitters/tika-emitter-opensearch/pom.xml | 135 --
.../pipes/emitter/opensearch/JsonResponse.java | 60 -
.../pipes/emitter/opensearch/OpenSearchClient.java | 336 ----
.../emitter/opensearch/OpenSearchEmitter.java | 241 ---
.../emitter/opensearch/OpenSearchClientTest.java | 52 -
.../test/resources/tika-config-simple-emitter.xml | 41 -
tika-pipes/tika-emitters/tika-emitter-s3/pom.xml | 112 --
.../apache/tika/pipes/emitter/s3/S3Emitter.java | 335 ----
tika-pipes/tika-emitters/tika-emitter-solr/pom.xml | 117 --
.../tika/pipes/emitter/solr/SolrEmitter.java | 360 ----
.../pipes/emitter/solr/SolrEmitterDevTest.java | 63 -
.../src/test/resources/log4j2.xml | 35 -
.../test/resources/tika-config-simple-emitter.xml | 48 -
tika-pipes/tika-fetchers/pom.xml | 48 -
.../tika-fetchers/tika-fetcher-az-blob/pom.xml | 119 --
.../tika/pipes/fetcher/azblob/AZBlobFetcher.java | 206 --
.../fetcher/azblob/config/AZBlobFetcherConfig.java | 72 -
.../pipes/fetcher/azblob/TestAZBlobFetcher.java | 55 -
.../src/test/resources/tika-config-az-blob.xml | 30 -
tika-pipes/tika-fetchers/tika-fetcher-gcs/pom.xml | 106 -
.../apache/tika/pipes/fetcher/gcs/GCSFetcher.java | 145 --
.../pipes/fetcher/gcs/config/GCSFetcherConfig.java | 62 -
.../tika/pipes/fetcher/s3/TestGCSFetcher.java | 63 -
.../src/test/resources/tika-config-gcs.xml | 26 -
tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml | 139 --
.../tika/pipes/fetcher/http/HttpFetcher.java | 690 -------
.../pipes/fetcher/http/config/HttpHeaders.java | 69 -
.../tika/pipes/fetcher/http/HttpFetcherTest.java | 301 ---
.../pipes/fetcher/http/config/HttpHeadersTest.java | 42 -
.../pipes/fetcher/http/jwt/JwtGeneratorTest.java | 56 -
.../src/test/resources/tika-config-http.xml | 32 -
.../tika-fetcher-microsoft-graph/pom.xml | 200 --
.../microsoftgraph/MicrosoftGraphFetcher.java | 140 --
.../config/AadCredentialConfigBase.java | 27 -
.../Client2CertificateCredentialsConfig.java | 50 -
.../config/ClientCertificateCredentialsConfig.java | 64 -
.../config/ClientSecretCredentialsConfig.java | 54 -
.../microsoftgraph/MicrosoftGraphFetcherTest.java | 105 -
tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml | 111 --
.../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 416 ----
.../pipes/fetcher/s3/config/S3FetcherConfig.java | 172 --
.../tika/pipes/fetcher/s3/TestS3Fetcher.java | 64 -
.../src/test/resources/tika-config-s3.xml | 27 -
tika-pipes/tika-httpclient-commons/pom.xml | 64 -
.../org/apache/tika/client/HttpClientFactory.java | 474 -----
.../org/apache/tika/client/HttpClientUtil.java | 76 -
.../apache/tika/client/TikaClientException.java | 29 -
tika-pipes/tika-pipes-cli/pom.xml | 112 ++
.../pipes/cli/filesystem/FileSystemFetcherCli.java | 163 ++
.../cli/googledrive/GoogleDriveFetcherCli.java | 143 ++
.../apache/tika/pipes/cli/http/HttpFetcherCli.java | 130 ++
.../pipes/cli/mapper/ObjectMapperProvider.java | 11 +
.../microsoftgraph/MicrosoftGraphFetcherCli.java | 172 ++
.../org/apache/tika/pipes/cli/s3/S3FetcherCli.java | 232 +++
.../tika-pipes-cli}/src/main/resources/log4j2.xml | 0
tika-pipes/tika-pipes-core/pom.xml | 136 +-
.../tika/pipes/core/CompositePipesReporter.java | 119 --
.../pipes/core/FailedToStartClientException.java | 27 -
.../org/apache/tika/pipes/core/FetchEmitTuple.java | 126 --
.../org/apache/tika/pipes/core/HandlerConfig.java | 172 --
.../org/apache/tika/pipes/core/PipesClient.java | 568 ------
.../org/apache/tika/pipes/core/PipesConfig.java | 66 -
.../apache/tika/pipes/core/PipesConfigBase.java | 192 --
.../org/apache/tika/pipes/core/PipesException.java | 27 -
.../org/apache/tika/pipes/core/PipesParser.java | 75 -
.../org/apache/tika/pipes/core/PipesReporter.java | 98 -
.../apache/tika/pipes/core/PipesReporterBase.java | 155 --
.../org/apache/tika/pipes/core/PipesResult.java | 114 --
.../org/apache/tika/pipes/core/PipesServer.java | 894 ---------
.../apache/tika/pipes/core/async/AsyncConfig.java | 120 --
.../apache/tika/pipes/core/async/AsyncEmitter.java | 144 --
.../tika/pipes/core/async/AsyncProcessor.java | 337 ----
.../apache/tika/pipes/core/async/AsyncStatus.java | 90 -
.../pipes/core/async/OfferLargerThanQueueSize.java | 41 -
.../tika/pipes/core/emitter/AbstractEmitter.java | 52 -
.../apache/tika/pipes/core/emitter/EmitData.java | 97 -
.../apache/tika/pipes/core/emitter/EmitKey.java | 80 -
.../apache/tika/pipes/core/emitter/Emitter.java | 35 -
.../tika/pipes/core/emitter/EmitterManager.java | 98 -
.../tika/pipes/core/emitter/StreamEmitter.java | 28 -
.../pipes/core/emitter/TikaEmitterException.java | 29 -
.../pipes/core/emitter/fs/FileSystemEmitter.java | 173 --
.../pipes/core/exception/TikaPipesException.java | 22 +
.../core/exception/TikaServerParseException.java | 22 +
.../AbstractEmbeddedDocumentBytesHandler.java | 78 -
.../BasicEmbeddedDocumentBytesHandler.java | 58 -
.../extractor/EmbeddedDocumentBytesConfig.java | 178 --
.../EmittingEmbeddedDocumentBytesHandler.java | 77 -
.../apache/tika/pipes/core/fetcher/FetchKey.java | 97 -
.../apache/tika/pipes/core/fetcher/Fetcher.java | 38 -
.../tika/pipes/core/fetcher/FetcherManager.java | 96 -
.../pipes/core/fetcher/FetcherStringException.java | 29 -
.../tika/pipes/core/fetcher/RangeFetcher.java | 40 -
.../pipes/core/fetcher/config/AbstractConfig.java | 21 -
.../fetcher/config/FetcherConfigContainer.java | 40 -
.../pipes/core/fetcher/fs/FileSystemFetcher.java | 182 --
.../fetcher/fs/config/FileSystemFetcherConfig.java | 42 -
.../tika/pipes/core/fetcher/url/UrlFetcher.java | 53 -
.../tika/pipes/core/parser/ParseService.java | 125 ++
.../core/pipesiterator/CallablePipesIterator.java | 125 --
.../pipes/core/pipesiterator/PipesIterator.java | 274 ---
.../pipes/core/pipesiterator/TotalCountResult.java | 54 -
.../pipes/core/pipesiterator/TotalCounter.java | 44 -
.../filelist/FileListPipesIterator.java | 107 -
.../pipesiterator/fs/FileSystemPipesIterator.java | 256 ---
.../pipes/core/serialization/JsonEmitData.java | 59 -
.../core/serialization/JsonFetchEmitTuple.java | 195 --
.../core/serialization/JsonFetchEmitTupleList.java | 75 -
.../apache/tika/pipes/core/util/CharsetUtil.java | 35 +
.../org/apache/tika/pipes/core/util/OSUtils.java | 258 +++
.../org/apache/tika/pipes/core/util/Props.java | 28 +
.../resources/pipes-fork-server-default-log4j2.xml | 32 -
.../apache/tika/pipes/core/PipesClientTest.java | 97 -
.../apache/tika/pipes/core/PipesServerTest.java | 192 --
.../tika/pipes/core/TikaPipesConfigTest.java | 123 --
.../pipes/core/async/AsyncChaosMonkeyTest.java | 202 --
.../tika/pipes/core/async/MockDigesterFactory.java | 56 -
.../apache/tika/pipes/core/async/MockEmitter.java | 59 -
.../apache/tika/pipes/core/async/MockFetcher.java | 44 -
.../apache/tika/pipes/core/async/MockReporter.java | 60 -
.../tika/pipes/core/async/MockReporterTest.java | 52 -
.../tika/pipes/core/emitter/MockEmitter.java | 60 -
.../tika/pipes/core/fetcher/MockFetcher.java | 72 -
.../core/fetcher/fs/FileSystemFetcherTest.java | 57 -
.../pipesiterator/FileSystemPipesIteratorTest.java | 81 -
.../filelist/FileListPipesIteratorTest.java | 79 -
.../serialization/JsonFetchEmitTupleListTest.java | 60 -
.../core/serialization/JsonFetchEmitTupleTest.java | 103 -
.../test/resources/configs/TIKA-4207-emitter.xml | 35 -
.../test/resources/configs/tika-config-broken.xml | 32 -
.../org/apache/tika/config/TIKA-3865-params.xml | 41 -
.../org/apache/tika/config/emitters-config.xml | 27 -
.../tika/config/emitters-duplicate-config.xml | 31 -
.../tika/config/emitters-mock-throw-on-check.xml | 27 -
.../org/apache/tika/config/fetchers-config.xml | 29 -
.../tika/config/fetchers-duplicate-config.xml | 33 -
.../tika/config/fetchers-mock-throw-on-check.xml | 27 -
.../tika/config/fetchers-nobasepath-config.xml | 28 -
.../apache/tika/config/fetchers-noname-config.xml | 32 -
.../apache/tika/config/pipes-iterator-config.xml | 25 -
.../tika/config/pipes-iterator-multiple-config.xml | 31 -
.../org/apache/tika/pipes/core/TIKA-3941.xml | 30 -
.../tika/pipes/core/TIKA-4207-limit-bytes.xml | 34 -
.../org/apache/tika/pipes/core/TIKA-4207.xml | 30 -
.../org/apache/tika/pipes/core/async/TIKA-3507.xml | 30 -
.../org/apache/tika/pipes/core/async/TIKA-3865.xml | 35 -
.../apache/tika/pipes/core/tika-sample-config.xml | 41 -
.../resources/test-documents/basic_embedded.xml | 35 -
.../test/resources/test-documents/file-list.txt | 10 -
.../resources/test-documents/mock/embedded.xml | 53 -
.../test/resources/test-documents/mock_times.xml | 26 -
.../test-documents/testOverlappingText.pdf | Bin 899 -> 0 bytes
tika-pipes/tika-pipes-emitters/README.md | 6 +
tika-pipes/tika-pipes-emitters/pom.xml | 109 +
.../tika-emitter-file-system/pom.xml | 82 +
.../src/main/assembly/assembly.xml | 30 +
.../fetchers/filesystem/FileSystemEmitter.java | 84 +
.../filesystem/FileSystemEmitterConfig.java | 45 +
.../filesystem/FileSystemEmitterPlugin.java} | 23 +-
.../src/main/resources/plugin.properties} | 13 +-
.../tika-pipes-emitters/tika-emitters-core/pom.xml | 38 +
.../pipes/core/emitter/DefaultEmitterConfig.java | 39 +
.../apache/tika/pipes/core/emitter/EmitOutput.java | 17 +
.../apache/tika/pipes/core/emitter/Emitter.java | 12 +
.../tika/pipes/core/emitter/EmitterConfig.java | 12 +
.../tika/pipes/core/emitter/OnExistBehavior.java | 5 +
.../pipes/core/emitter/TikaEmitterException.java | 11 +
tika-pipes/tika-pipes-fetchers/README.md | 53 +
tika-pipes/tika-pipes-fetchers/pom.xml | 114 ++
.../tika-fetcher-atlassian-jwt/pom.xml | 117 ++
.../src/main/assembly/assembly.xml | 30 +
.../fetchers/atlassianjwt/AtlassianJwtFetcher.java | 347 ++++
.../atlassianjwt/AtlassianJwtFetcherPlugin.java} | 30 +-
.../atlassianjwt/AtlassianJwtGenerator.java | 189 ++
.../config/AtlassianJwtFetcherConfig.java | 191 ++
.../src/main/resources/plugin.properties | 6 +
.../tika-fetcher-file-system/pom.xml | 82 +
.../src/main/assembly/assembly.xml | 30 +
.../fetchers/filesystem/FileSystemFetcher.java | 100 +
.../filesystem/FileSystemFetcherConfig.java | 29 +
.../filesystem/FileSystemFetcherPlugin.java} | 29 +-
.../src/main/resources/plugin.properties | 6 +
.../tika-fetcher-google-drive/pom.xml | 132 ++
.../src/main/assembly/assembly.xml | 30 +
.../fetchers/googledrive/GoogleDriveFetcher.java | 122 ++
.../fetchers/googledrive/GoogleDrivePlugin.java} | 27 +-
.../config/GoogleDriveFetcherConfig.java | 56 +
.../src/main/resources/plugin.properties | 6 +
.../src/test/resources/log4j2.xml | 0
.../tika-pipes-fetchers/tika-fetcher-http/pom.xml | 117 ++
.../src/main/assembly/assembly.xml | 30 +
.../tika/pipes/fetchers/http/HttpFetcher.java | 406 ++++
.../pipes/fetchers/http/HttpFetcherPlugin.java} | 27 +-
.../pipes/fetchers/http/config/AuthConfig.java | 54 +
.../fetchers}/http/config/HttpFetcherConfig.java | 152 +-
.../tika/pipes/fetchers/http/config/JwtConfig.java | 54 +
.../pipes/fetchers/http/config/ProxyConfig.java | 24 +
.../tika/pipes/fetchers}/http/jwt/JwtCreds.java | 2 +-
.../pipes/fetchers}/http/jwt/JwtGenerator.java | 2 +-
.../fetchers}/http/jwt/JwtPrivateKeyCreds.java | 2 +-
.../pipes/fetchers}/http/jwt/JwtSecretCreds.java | 2 +-
.../src/main/resources/plugin.properties | 6 +
.../tika-fetcher-microsoft-graph/pom.xml | 197 ++
.../src/main/assembly/assembly.xml | 30 +
.../microsoftgraph/MicrosoftGraphFetcher.java | 128 ++
.../microsoftgraph/MicrosoftGraphPlugin.java} | 27 +-
.../config/MicrosoftGraphFetcherConfig.java | 70 +-
.../src/main/resources/plugin.properties | 6 +
.../src/test/resources/log4j2.xml | 0
.../tika-pipes-fetchers/tika-fetcher-s3/pom.xml | 127 ++
.../tika-fetcher-s3/src/main/assembly/assembly.xml | 30 +
.../tika/pipes/fetchers/s3/S3ClientManager.java | 88 +
.../apache/tika/pipes/fetchers/s3/S3Fetcher.java | 181 ++
.../apache/tika/pipes/fetchers/s3/S3Plugin.java} | 29 +-
.../pipes/fetchers/s3/config/S3FetcherConfig.java | 67 +
.../src/main/resources/plugin.properties | 6 +
.../tika-fetcher-s3}/src/test/resources/log4j2.xml | 0
.../tika-pipes-fetchers/tika-fetchers-core/pom.xml | 38 +
.../pipes/fetchers/core/DefaultFetcherConfig.java | 39 +
.../apache/tika/pipes/fetchers/core/Fetcher.java | 10 +
.../tika/pipes/fetchers/core/FetcherConfig.java | 14 +
tika-pipes/tika-pipes-grpc/README.md | 17 +
.../tika-pipes-grpc/corpa-files}/014760.docx | Bin
.../tika-pipes-grpc/corpa-files}/017091.docx | Bin
.../tika-pipes-grpc/corpa-files}/017097.docx | Bin
.../tika-pipes-grpc/corpa-files}/018367.docx | Bin
tika-pipes/tika-pipes-grpc/corpa-files/018370.docx | Bin 0 -> 30723 bytes
tika-pipes/tika-pipes-grpc/corpa-files/018371.docx | Bin 0 -> 26036 bytes
tika-pipes/tika-pipes-grpc/corpa-files/021524.docx | Bin 0 -> 16028 bytes
.../corpa-files/file-that-confuses-tika.log | 419 ++++
tika-pipes/tika-pipes-grpc/docker-build/Dockerfile | 40 +
.../tika-pipes-grpc/docker-build/docker-build.sh | 119 ++
.../docker-build/start-tika-grpc.sh | 33 +
tika-pipes/tika-pipes-grpc/pom.xml | 388 ++++
.../apache/tika/pipes/TikaPipesApplication.java | 12 +
.../tika/pipes/grpc/TikaGrpcServerCustomizer.java | 19 +
.../apache/tika/pipes/grpc/TikaGrpcService.java | 754 +++++++
.../java/org/apache/tika/pipes/job/JobStatus.java | 19 +
.../tika/pipes/model/FetchAndParseStatus.java | 6 +
.../parser/TikaPipesApplicationConfiguration.java | 49 +
.../plugin/ClasspathPluginPropertiesFinder.java | 18 +
.../tika/pipes/plugin/GrpcPluginManager.java | 100 +
.../org/apache/tika/pipes/plugin/PluginConfig.java | 28 +
.../apache/tika/pipes/repo/EmitterRepository.java | 19 +
.../apache/tika/pipes/repo/FetcherRepository.java | 19 +
.../pipes/repo/IgniteRepositoryConfiguration.java | 69 +
.../tika/pipes/repo/JobStatusRepository.java | 18 +
.../tika/pipes/repo/PipeIteratorRepository.java | 19 +
.../src/main/resources/application.yaml | 25 +
.../tika/pipes/TikaPipesIntegrationTestBase.java | 72 +
.../pipes/grpc/TikaGrpcServerEmittersCrudTest.java | 79 +
.../grpc/TikaGrpcServerFetchAndParseTest.java | 166 ++
.../pipes/grpc/TikaGrpcServerFetchersCrudTest.java | 77 +
.../pipes/grpc/TikaGrpcServerJsonSchemaTest.java | 77 +
.../grpc/TikaGrpcServerPipeIteratorsCrudTest.java | 81 +
.../tika/pipes/grpc/TikaGrpcServerPipeJobTest.java | 156 ++
tika-pipes/tika-pipes-iterators/pom.xml | 50 -
.../tika-pipes-iterator-az-blob/pom.xml | 108 -
.../pipesiterator/azblob/AZBlobPipesIterator.java | 151 --
.../azblob/TestAZBlobPipesIterator.java | 99 -
.../tika-pipes-iterator-csv/pom.xml | 118 --
.../pipes/pipesiterator/csv/CSVPipesIterator.java | 254 ---
.../src/test/java/TestCSVPipesIterator.java | 131 --
.../src/test/resources/test-simple.csv | 6 -
.../tika-pipes-iterator-gcs/pom.xml | 108 -
.../pipes/pipesiterator/gcs/GCSPipesIterator.java | 126 --
.../pipesiterator/gcs/TestGCSPipesIterator.java | 101 -
.../tika-pipes-iterator-jdbc/pom.xml | 109 -
.../pipesiterator/jdbc/JDBCPipesIterator.java | 379 ----
.../pipesiterator/jdbc/TestJDBCPipesIterator.java | 188 --
.../src/test/resources/log4j2.xml | 32 -
.../tika-pipes-iterator-json/pom.xml | 120 --
.../pipesiterator/json/JsonPipesIterator.java | 65 -
.../pipesiterator/json/TestJsonPipesIterator.java | 92 -
.../test-documents/test-with-embedded-bytes.json | 2100 --------------------
.../src/test/resources/test-documents/test.json | 1400 -------------
.../tika-pipes-iterator-kafka/pom.xml | 117 --
.../pipesiterator/kafka/KafkaPipesIterator.java | 174 --
.../kafka/TestKafkaPipesIterator.java | 99 -
.../src/test/resources/log4j2.xml | 32 -
.../tika-pipes-iterator-s3/pom.xml | 113 --
.../pipes/pipesiterator/s3/S3PipesIterator.java | 217 --
.../pipesiterator/s3/TestS3PipesIterator.java | 100 -
.../src/test/resources/log4j2.xml | 32 -
.../tika-pipes-iterator-solr/pom.xml | 114 --
.../pipesiterator/solr/SolrPipesIterator.java | 277 ---
tika-pipes/tika-pipes-pipe-iterators/README.md | 11 +
tika-pipes/tika-pipes-pipe-iterators/pom.xml | 109 +
.../tika-pipe-iterator-csv/pom.xml | 94 +
.../src/main/assembly/assembly.xml | 30 +
.../pipes/fetchers/filesystem/CsvPipeIterator.java | 90 +
.../fetchers/filesystem/CsvPipeIteratorConfig.java | 21 +
.../filesystem/CsvPipeIteratorPlugin.java} | 32 +-
.../src/main/resources/plugin.properties | 6 +
.../tika-pipe-iterators-core/pom.xml | 39 +
.../core/iterators/DefaultPipeIteratorConfig.java | 44 +
.../tika/pipes/core/iterators/PipeInput.java | 15 +
.../tika/pipes/core/iterators/PipeIterator.java | 12 +
.../pipes/core/iterators/PipeIteratorConfig.java | 14 +
.../core/iterators/TikaPipeIteratorException.java | 11 +
tika-pipes/tika-pipes-proto/pom.xml | 184 ++
.../src/main/proto/google/api/annotations.proto | 31 +
.../src/main/proto/google/api/http.proto | 371 ++++
.../tika-pipes-proto/src/main/proto/tika.proto | 440 ++++
tika-pipes/tika-pipes-reporters/pom.xml | 43 -
.../tika-pipes-reporter-fs-status/pom.xml | 135 --
.../reporters/fs/FileSystemStatusReporter.java | 199 --
.../reporters/fs/TestFileSystemStatusReporter.java | 181 --
.../tika-pipes-reporter-jdbc/pom.xml | 108 -
.../pipes/reporters/jdbc/JDBCPipesReporter.java | 490 -----
.../reporters/jdbc/TestJDBCPipesReporter.java | 297 ---
.../resources/configs/tika-config-advanced.xml | 50 -
.../resources/configs/tika-config-excludes.xml | 42 -
.../resources/configs/tika-config-includes.xml | 42 -
.../tika-pipes-reporter-opensearch/pom.xml | 113 --
.../pipes/reporters/opensearch/JsonResponse.java | 60 -
.../reporters/opensearch/OpenSearchClient.java | 168 --
.../opensearch/OpenSearchPipesReporter.java | 243 ---
{tika-server => tika-server-old}/README.md | 0
{tika-server => tika-server-old}/pom.xml | 21 +
.../tika-server-client/pom.xml | 2 +
.../tika/server/client/TikaAsyncHttpClient.java | 0
.../org/apache/tika/server/client/TikaClient.java | 0
.../apache/tika/server/client/TikaClientCLI.java | 0
.../server/client/TikaClientConfigException.java | 0
.../tika/server/client/TikaEmitterResult.java | 0
.../tika/server/client/TikaPipesHttpClient.java | 0
.../tika/server/client/TikaServerClientConfig.java | 0
.../org/apache/tika/server/client/TestBasic.java | 0
.../resources/tika-config-simple-fs-emitter.xml | 0
.../tika-server-core/pom.xml | 2 +
.../server/core/CompositeParseContextConfig.java | 0
.../server/core/DefaultInputStreamFactory.java | 0
.../tika/server/core/FetcherStreamFactory.java | 0
.../org/apache/tika/server/core/HTMLHelper.java | 0
.../tika/server/core/InputStreamFactory.java | 0
.../org/apache/tika/server/core/MetadataList.java | 0
.../tika/server/core/ParseContextConfig.java | 0
.../server/core/ProduceTypeResourceComparator.java | 0
.../org/apache/tika/server/core/ServerStatus.java | 0
.../tika/server/core/ServerStatusResource.java | 0
.../tika/server/core/ServerStatusWatcher.java | 0
.../org/apache/tika/server/core/TaskStatus.java | 0
.../apache/tika/server/core/TikaLoggingFilter.java | 0
.../org/apache/tika/server/core/TikaServerCli.java | 0
.../apache/tika/server/core/TikaServerConfig.java | 0
.../tika/server/core/TikaServerParseException.java | 0
.../core/TikaServerParseExceptionMapper.java | 0
.../apache/tika/server/core/TikaServerProcess.java | 0
.../tika/server/core/TikaServerWatchDog.java | 0
.../org/apache/tika/server/core/TlsConfig.java | 0
.../apache/tika/server/core/WatchDogResult.java | 0
.../server/core/config/DocumentSelectorConfig.java | 0
.../server/core/config/PasswordProviderConfig.java | 0
.../tika/server/core/config/TimeoutConfig.java | 0
.../tika/server/core/resource/AsyncRequest.java | 0
.../tika/server/core/resource/AsyncResource.java | 0
.../server/core/resource/DetectorResource.java | 0
.../server/core/resource/LanguageResource.java | 0
.../server/core/resource/MetadataResource.java | 0
.../tika/server/core/resource/PipesResource.java | 0
.../core/resource/RecursiveMetadataResource.java | 0
.../tika/server/core/resource/TikaDetectors.java | 0
.../tika/server/core/resource/TikaMimeTypes.java | 0
.../tika/server/core/resource/TikaParsers.java | 0
.../tika/server/core/resource/TikaResource.java | 0
.../server/core/resource/TikaServerResource.java | 0
.../server/core/resource/TikaServerStatus.java | 0
.../tika/server/core/resource/TikaVersion.java | 0
.../tika/server/core/resource/TikaWelcome.java | 0
.../server/core/resource/TranslateResource.java | 0
.../server/core/resource/UnpackerResource.java | 0
.../server/core/writer/CSVMessageBodyWriter.java | 0
.../server/core/writer/JSONMessageBodyWriter.java | 0
.../tika/server/core/writer/JSONObjWriter.java | 0
.../core/writer/MetadataListMessageBodyWriter.java | 0
.../apache/tika/server/core/writer/TarWriter.java | 0
.../server/core/writer/TextMessageBodyWriter.java | 0
.../tika/server/core/writer/TikaServerWriter.java | 0
.../apache/tika/server/core/writer/ZipWriter.java | 0
.../org.apache.tika.server.core.ParseContextConfig | 0
.../main/resources/tika-server-config-default.xml | 0
.../src/main/resources/tikaserver-template.html | 0
.../main/resources/tikaserver-version.properties | 0
.../org/apache/tika/server/core/CXFTestBase.java | 0
.../tika/server/core/IntegrationTestBase.java | 0
.../tika/server/core/LanguageResourceTest.java | 0
.../tika/server/core/NullWebClientLogger.java | 0
.../server/core/RecursiveMetadataResourceTest.java | 0
.../apache/tika/server/core/ServerStatusTest.java | 0
.../apache/tika/server/core/StackTraceOffTest.java | 0
.../apache/tika/server/core/StackTraceTest.java | 0
.../apache/tika/server/core/TikaMimeTypesTest.java | 0
.../org/apache/tika/server/core/TikaPipesTest.java | 0
.../tika/server/core/TikaResourceFetcherTest.java | 0
.../core/TikaResourceMetadataFilterTest.java | 0
.../tika/server/core/TikaResourceNoStackTest.java | 0
.../apache/tika/server/core/TikaResourceTest.java | 0
.../core/TikaServerAsyncIntegrationTest.java | 0
.../tika/server/core/TikaServerConfigTest.java | 0
.../server/core/TikaServerIntegrationTest.java | 0
.../core/TikaServerPipesIntegrationTest.java | 0
.../tika/server/core/TikaServerStatusTest.java | 0
.../apache/tika/server/core/TikaVersionTest.java | 0
.../apache/tika/server/core/TikaWelcomeTest.java | 0
.../tika/server/core/TranslateResourceTest.java | 0
.../resources/configs/metadata-filter-include.xml | 0
.../configs/tika-config-server-badjvmargs.xml | 0
.../resources/configs/tika-config-server-basic.xml | 0
.../configs/tika-config-server-emitter.xml | 0
.../tika-config-server-fetcher-template.xml | 0
.../tika-config-server-fetchers-emitters.xml | 0
.../configs/tika-config-server-timeout-10000.xml | 0
.../tika-config-server-tls-one-way-template.xml | 0
.../tika-config-server-tls-two-way-template.xml | 0
.../resources/configs/tika-config-server-tls.xml | 0
.../test/resources/configs/tika-config-server.xml | 0
.../src/test/resources/logging/log4j2_forked.xml | 0
.../src/test/resources/ssl-keys/README.txt | 0
.../resources/ssl-keys/tika-client-keystore.p12 | Bin
.../resources/ssl-keys/tika-client-truststore.p12 | Bin
.../src/test/resources/ssl-keys/tika-client.crt | Bin
.../resources/ssl-keys/tika-server-keystore.p12 | Bin
.../resources/ssl-keys/tika-server-truststore.p12 | Bin
.../src/test/resources/ssl-keys/tika-server.crt | Bin
.../src/test/resources/test-documents/english.txt | 0
.../src/test/resources/test-documents/french.txt | 0
.../mock/encrypted_document_exception.xml | 0
.../resources/test-documents/mock/fake_oom.xml | 0
.../test-documents/mock/heavy_hang_100.xml | 0
.../test-documents/mock/heavy_hang_30000.xml | 0
.../resources/test-documents/mock/hello_world.xml | 0
.../test-documents/mock/hello_world_long.xml | 0
.../resources/test-documents/mock/null_pointer.xml | 0
.../resources/test-documents/mock/real_oom.xml | 0
.../resources/test-documents/mock/system_exit.xml | 0
.../test-documents/mock/testStaticStdOutErr.xml | 0
.../test-documents/mock/testStdOutErr.xml | 0
.../test-documents/mock/thread_interrupt.xml | 0
.../mock/\344\270\255\346\226\207.xml" | 0
.../test/resources/test-documents/testDigilite.fdf | 0
.../tika-server-eval/pom.xml | 0
.../apache/tika/server/eval/TikaEvalResource.java | 0
...he.tika.server.core.resource.TikaServerResource | 0
.../tika/server/eval/TikaEvalResourceTest.java | 0
tika-server-old/tika-server-spring/pom.xml | 170 ++
.../apache/tika/server/TikaServerApplication.java | 55 +
.../tika/server/controller/AsyncController.java | 182 ++
.../tika/server/controller/DetectorController.java | 84 +
.../controller/InformationServicesController.java | 282 +++
.../controller/LanguageResourceController.java | 142 ++
.../tika/server/controller/MetadataController.java | 177 ++
.../tika/server/controller/PipesController.java | 194 ++
.../RecursiveMetadataAndContentController.java | 240 +++
.../server/controller/ServerStatusController.java | 101 +
.../tika/server/controller/TikaController.java | 292 +++
.../controller/TranslateResourceController.java | 118 ++
.../controller/UnpackResourceController.java | 249 +++
.../main/resources/api/tika-server-openapi.yaml | 1098 ++++++++++
.../src/main/resources/application.properties | 30 +-
.../tika-server-standard/assembly.xml | 0
.../tika-server-standard/bin/init.d/tika | 0
.../bin/install_tika_service.sh | 0
.../tika-server-standard/bin/tika | 0
.../tika-server-standard/bin/tika.in.sh | 0
.../tika-server-standard/pom.xml | 0
.../server/standard/config/PDFServerConfig.java | 0
.../standard/config/TesseractServerConfig.java | 0
.../standard/resource/XMPMetadataResource.java | 0
.../standard/writer/XMPMessageBodyWriter.java | 0
.../services/org.apache.tika.parser.Parser | 0
.../org.apache.tika.server.core.ParseContextConfig | 0
...he.tika.server.core.resource.TikaServerResource | 0
...apache.tika.server.core.writer.TikaServerWriter | 0
.../tika/server/standard/DetectorResourceTest.java | 0
.../apache/tika/server/standard/FetcherTest.java | 0
.../server/standard/JsonMaxFieldLengthTest.java | 0
.../tika/server/standard/MetadataResourceTest.java | 0
.../server/standard/OpenNLPMetadataFilterTest.java | 0
.../standard/OptimaizeMetadataFilterTest.java | 0
.../standard/RecursiveMetadataFilterTest.java | 0
.../standard/RecursiveMetadataResourceTest.java | 0
.../tika/server/standard/TikaDetectorsTest.java | 0
.../tika/server/standard/TikaMimeTypesTest.java | 0
.../tika/server/standard/TikaParsersTest.java | 0
.../apache/tika/server/standard/TikaPipesTest.java | 0
.../tika/server/standard/TikaResourceTest.java | 0
.../tika/server/standard/UnpackerResourceTest.java | 0
.../standard/UnpackerResourceWithConfigTest.java | 0
.../test/resources/config/TIKA-3137-include.xml | 0
.../config/tika-config-for-server-tests.xml | 0
.../src/test/resources/config/tika-config-json.xml | 0
.../tika-config-langdetect-opennlp-filter.xml | 0
.../tika-config-langdetect-optimaize-filter.xml | 0
.../test/resources/config/tika-config-unpacker.xml | 0
.../resources/config/tika-config-url-fetcher.xml | 0
.../src/test/resources/test-documents/2exe.docx | Bin
.../src/test/resources/test-documents/2pic.doc | Bin
.../src/test/resources/test-documents/2pic.docx | Bin
.../test-documents/CDEC_WEATHER_2010_03_02 | 0
.../src/test/resources/test-documents/Doc1_ole.doc | Bin
.../src/test/resources/test-documents/foo.csv | 0
.../src/test/resources/test-documents/password.xls | Bin
.../src/test/resources/test-documents/pic.xls | Bin
.../src/test/resources/test-documents/pic.xlsx | Bin
.../src/test/resources/test-documents/test.doc | Bin
.../test-documents/testColorRendering.pdf | Bin
.../test/resources/test-documents/testHTML.html | 0
.../src/test/resources/test-documents/testOCR.pdf | Bin
.../test-documents/testPDFTwoTextBoxes.pdf | Bin
.../test-documents/testPDF_XFA_govdocs1_258578.pdf | Bin
.../test-documents/testPassword4Spaces.pdf | Bin
.../testRTF_npeFromWMFInTikaServer.rtf | 0
.../test-documents/testUnicodePassword.pdf | Bin
.../test-documents/test_recursive_embedded.docx | Bin
.../src/main/resources/log4j2.xml | 32 -
.../src/test/resources/log4j2.xml | 32 -
.../tika-server-core/src/main/resources/log4j2.xml | 32 -
.../tika-server-core/src/test/resources/log4j2.xml | 32 -
568 files changed, 13433 insertions(+), 29825 deletions(-)
diff --git a/pom.xml b/pom.xml
index 58635e421..be20e234d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,7 @@
<module>tika-xmp</module>
<module>tika-langdetect</module>
<module>tika-pipes</module>
- <module>tika-grpc</module>
+ <module>tika-server-spring</module>
<module>tika-app</module>
<module>tika-server</module>
<module>tika-integration-tests</module>
diff --git a/tika-grpc/README.md b/tika-grpc/README.md
deleted file mode 100644
index f2269b22b..000000000
--- a/tika-grpc/README.md
+++ /dev/null
@@ -1,13 +0,0 @@
-# Tika Pipes GRPC Server
-
-The following is the Tika Pipes GRPC Server.
-
-This server will manage a pool of Tika Pipes clients.
-
-* Tika Pipes Fetcher CRUD operations
- * Create
- * Read
- * Update
- * Delete
-* Fetch + Parse a given Fetch Item
-
diff --git a/tika-grpc/pom.xml b/tika-grpc/pom.xml
deleted file mode 100644
index cf3433da3..000000000
--- a/tika-grpc/pom.xml
+++ /dev/null
@@ -1,438 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>tika-grpc</artifactId>
- <packaging>jar</packaging>
- <!-- Feel free to delete the comment at the end of these lines. It is just
- for safely updating the version in our release process. -->
- <name>Apache Tika pipes gRPC server</name>
- <url>https://tika.apache.org/</url>
-
- <parent>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-parent</artifactId>
- <version>4.0.0-SNAPSHOT</version>
- <relativePath>../tika-parent/pom.xml</relativePath>
- </parent>
-
- <properties>
- <grpc.version>1.75.0</grpc.version>
- <asarkar-grpc-test.version>2.0.0</asarkar-grpc-test.version>
- <awaitility.version>4.3.0</awaitility.version>
- <j2objc-annotations.version>3.1</j2objc-annotations.version>
- </properties>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-bom</artifactId>
- <version>${grpc.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
- <dependencies>
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-netty-shaded</artifactId>
- <scope>runtime</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.errorprone</groupId>
- <artifactId>error_prone_annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.errorprone</groupId>
- <artifactId>error_prone_annotations</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-services</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.errorprone</groupId>
- <artifactId>error_prone_annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.j2objc</groupId>
- <artifactId>j2objc-annotations</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-stub</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.errorprone</groupId>
- <artifactId>error_prone_annotations</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java-util</artifactId>
- <version>${protobuf.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.errorprone</groupId>
- <artifactId>error_prone_annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.j2objc</groupId>
- <artifactId>j2objc-annotations</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.google.j2objc</groupId>
- <artifactId>j2objc-annotations</artifactId>
- <version>${j2objc-annotations.version}</version> <!-- prevent downgrade of version in guava -->
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-async-cli</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-parsers-standard-package</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.errorprone</groupId>
- <artifactId>error_prone_annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.j2objc</groupId>
- <artifactId>j2objc-annotations</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <!-- logging -->
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.tomcat</groupId>
- <artifactId>annotations-api</artifactId>
- <version>6.0.53</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-testing</artifactId>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-fetcher-http</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-jsonSchema</artifactId>
- </dependency>
- <dependency>
- <groupId>com.asarkar.grpc</groupId>
- <artifactId>grpc-test</artifactId>
- <version>${asarkar-grpc-test.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.awaitility</groupId>
- <artifactId>awaitility</artifactId>
- <version>${awaitility.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <extensions>
- <extension>
- <groupId>kr.motd.maven</groupId>
- <artifactId>os-maven-plugin</artifactId>
- <version>1.7.1</version>
- </extension>
- </extensions>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>${maven.javadoc.version}</version>
- <configuration>
- <doclint>none</doclint>
- <sourcepath>src/main/java;${project.build.directory}/generated-sources/protobuf/grpc-java:${project.build.directory}/generated-sources/protobuf/java</sourcepath>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.xolstice.maven.plugins</groupId>
- <artifactId>protobuf-maven-plugin</artifactId>
- <version>0.6.1</version>
- <configuration>
- <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
- <pluginId>grpc-java</pluginId>
- <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>compile</goal>
- <goal>compile-custom</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-enforcer-plugin</artifactId>
- <executions>
- <execution>
- <id>enforce</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <configuration>
- <rules>
- <requireUpperBoundDeps />
- </rules>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes>
- <exclude>src/test/resources/certs/**</exclude>
- <exclude>README.md</exclude>
- </excludes>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>3.6.1</version>
- <executions>
- <execution>
- <id>test</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>${basedir}/target/generated-sources/protobuf/grpc-java</source>
- <source>${basedir}/target/generated-sources/protobuf/java</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>3.5.1</version>
- <executions>
- <execution>
- <goals>
- <goal>java</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <mainClass>org.apache.tika.pipes.grpc.TikaGrpcServer</mainClass>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <artifactSet>
- <excludes>
- </excludes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>module-info.class</exclude>
- <exclude>META-INF/maven/plugin.xml</exclude>
- <exclude>META-INF/versions/9/module-info.class</exclude>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- <exclude>META-INF/*.txt</exclude>
- <exclude>META-INF/ASL2.0</exclude>
- <exclude>META-INF/DEPENDENCIES</exclude>
- <exclude>META-INF/LICENSE</exclude>
- <exclude>META-INF/NOTICE</exclude>
- <exclude>META-INF/README</exclude>
- <exclude>META-INF/MANIFEST.MF</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- <exclude>CHANGES</exclude>
- <exclude>README</exclude>
- <exclude>builddef.lst</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.tika.pipes.grpc.TikaGrpcServer</mainClass>
- <manifestEntries>
- <Multi-Release>true</Multi-Release>
- </manifestEntries>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <version>${checkstyle.plugin.version}</version>
- <dependencies>
- <dependency>
- <groupId>com.puppycrawl.tools</groupId>
- <artifactId>checkstyle</artifactId>
- <version>${puppycrawl.version}</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>validate</id>
- <phase>validate</phase>
- <configuration>
- <configLocation>checkstyle.xml</configLocation>
- <inputEncoding>UTF-8</inputEncoding>
- <consoleOutput>true</consoleOutput>
- <logViolationsToConsole>true</logViolationsToConsole>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <testSourceDirectories>${project.basedir}/src/test/java</testSourceDirectories>
- <violationSeverity>error</violationSeverity>
- <failOnViolation>true</failOnViolation>
- <excludeGeneratedSources>true</excludeGeneratedSources>
- </configuration>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/ExpiringFetcherStore.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/ExpiringFetcherStore.java
deleted file mode 100644
index 3dc10f09c..000000000
--- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/ExpiringFetcherStore.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.grpc;
-
-import java.time.Instant;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
-
-public class ExpiringFetcherStore implements AutoCloseable {
- private static final Logger LOG = LoggerFactory.getLogger(ExpiringFetcherStore.class);
- public static final long EXPIRE_JOB_INITIAL_DELAY = 1L;
- private final Map<String, AbstractFetcher> fetchers = Collections.synchronizedMap(new HashMap<>());
- private final Map<String, AbstractConfig> fetcherConfigs = Collections.synchronizedMap(new HashMap<>());
- private final Map<String, Instant> fetcherLastAccessed = Collections.synchronizedMap(new HashMap<>());
-
- private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
-
- public ExpiringFetcherStore(int expireAfterSeconds, int checkForExpiredFetchersDelaySeconds) {
- executorService.scheduleAtFixedRate(() -> {
- Set<String> expired = new HashSet<>();
- for (String fetcherName : fetchers.keySet()) {
- Instant lastAccessed = fetcherLastAccessed.get(fetcherName);
- if (lastAccessed == null) {
- LOG.error("Detected a fetcher with no last access time. FetcherName={}", fetcherName);
- expired.add(fetcherName);
- } else if (Instant
- .now()
- .isAfter(lastAccessed.plusSeconds(expireAfterSeconds))) {
- LOG.info("Detected stale fetcher {} hasn't been accessed in {} seconds. " + "Deleting.", fetcherName, Instant
- .now()
- .getEpochSecond() - lastAccessed.getEpochSecond());
- expired.add(fetcherName);
- }
- }
- for (String expiredFetcherId : expired) {
- deleteFetcher(expiredFetcherId);
- }
- }, EXPIRE_JOB_INITIAL_DELAY, checkForExpiredFetchersDelaySeconds, TimeUnit.SECONDS);
- }
-
- public boolean deleteFetcher(String fetcherName) {
- boolean success = fetchers.remove(fetcherName) != null;
- fetcherConfigs.remove(fetcherName);
- fetcherLastAccessed.remove(fetcherName);
- return success;
- }
-
- public Map<String, AbstractFetcher> getFetchers() {
- return fetchers;
- }
-
- public Map<String, AbstractConfig> getFetcherConfigs() {
- return fetcherConfigs;
- }
-
- /**
- * This method will get the fetcher, but will also log the access the fetcher as having
- * been accessed. This prevents the scheduled job from removing the stale fetcher.
- */
- public <T extends AbstractFetcher> T getFetcherAndLogAccess(String fetcherName) {
- fetcherLastAccessed.put(fetcherName, Instant.now());
- return (T) fetchers.get(fetcherName);
- }
-
- public <T extends AbstractFetcher, C extends AbstractConfig> void createFetcher(T fetcher, C config) {
- fetchers.put(fetcher.getName(), fetcher);
- fetcherConfigs.put(fetcher.getName(), config);
- getFetcherAndLogAccess(fetcher.getName());
- }
-
- @Override
- public void close() {
- executorService.shutdownNow();
- }
-}
diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServer.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServer.java
deleted file mode 100644
index c9be5fb33..000000000
--- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServer.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.grpc;
-
-import static io.grpc.health.v1.HealthCheckResponse.ServingStatus;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.nio.charset.StandardCharsets;
-import java.util.concurrent.TimeUnit;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import io.grpc.Grpc;
-import io.grpc.InsecureServerCredentials;
-import io.grpc.Server;
-import io.grpc.ServerCredentials;
-import io.grpc.TlsServerCredentials;
-import io.grpc.protobuf.services.HealthStatusManager;
-import io.grpc.protobuf.services.ProtoReflectionServiceV1;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.config.TikaConfigSerializer;
-
-/**
- * Server that manages startup/shutdown of the GRPC Tika server.
- */
-public class TikaGrpcServer {
- private static final Logger LOGGER = LoggerFactory.getLogger(TikaGrpcServer.class);
- public static final int TIKA_SERVER_GRPC_DEFAULT_PORT = 50052;
- private Server server;
- @Parameter(names = {"-p", "--port"}, description = "The grpc server port", help = true)
- private Integer port = TIKA_SERVER_GRPC_DEFAULT_PORT;
-
- @Parameter(names = {"-c", "--config"}, description = "The grpc server port", help = true)
- private File tikaConfigXml;
-
- @Parameter(names = {"-s", "--secure"}, description = "Enable credentials required to access this grpc server")
- private boolean secure;
-
- @Parameter(names = {"--cert-chain"}, description = "Certificate chain file. Example: server1.pem See: https://github.com/grpc/grpc-java/tree/b3ffb5078df361d7460786e134db7b5c00939246/examples/example-tls")
- private File certChain;
-
- @Parameter(names = {"--private-key"}, description = "Private key store. Example: server1.key See: https://github.com/grpc/grpc-java/tree/b3ffb5078df361d7460786e134db7b5c00939246/examples/example-tls")
- private File privateKey;
-
- @Parameter(names = {"--private-key-password"}, description = "Private key password, if needed")
- private String privateKeyPassword;
-
- @Parameter(names = {"--trust-cert-collection"}, description = "The trust certificate collection (root certs). Example: ca.pem See: https://github.com/grpc/grpc-java/tree/b3ffb5078df361d7460786e134db7b5c00939246/examples/example-tls")
- private File trustCertCollection;
-
- @Parameter(names = {"--client-auth-required"}, description = "Is Mutual TLS required?")
- private boolean clientAuthRequired;
-
- @Parameter(names = {"-h", "-H", "--help"}, description = "Display help menu")
- private boolean help;
-
- public void start() throws Exception {
- HealthStatusManager healthStatusManager = new HealthStatusManager();
- ServerCredentials creds;
- if (secure) {
- TlsServerCredentials.Builder channelCredBuilder = TlsServerCredentials.newBuilder();
- channelCredBuilder.keyManager(certChain, privateKey, privateKeyPassword);
- if (trustCertCollection != null && trustCertCollection.exists()) {
- channelCredBuilder.trustManager(trustCertCollection);
- if (clientAuthRequired) {
- channelCredBuilder.clientAuth(TlsServerCredentials.ClientAuth.REQUIRE);
- }
- }
- creds = channelCredBuilder.build();
- } else {
- creds = InsecureServerCredentials.create();
- }
- if (tikaConfigXml == null) {
- // Create a default tika config
- tikaConfigXml = File.createTempFile("tika-config", ".xml");
- try (FileWriter fw = new FileWriter(tikaConfigXml, StandardCharsets.UTF_8)) {
- TikaConfigSerializer.serialize(new TikaConfig(), TikaConfigSerializer.Mode.STATIC_FULL, fw, StandardCharsets.UTF_8);
- }
- }
- File tikaConfigFile = new File(tikaConfigXml.getAbsolutePath());
- healthStatusManager.setStatus(TikaGrpcServer.class.getSimpleName(), ServingStatus.SERVING);
- server = Grpc
- .newServerBuilderForPort(port, creds)
- .addService(new TikaGrpcServerImpl(tikaConfigFile.getAbsolutePath()))
- .addService(healthStatusManager.getHealthService())
- .addService(ProtoReflectionServiceV1.newInstance())
- .build()
- .start();
- LOGGER.info("Server started, listening on " + port);
- Runtime
- .getRuntime()
- .addShutdownHook(new Thread(() -> {
- // Use stderr here since the logger may have been reset by its JVM shutdown hook.
- System.err.println("*** shutting down gRPC server since JVM is shutting down");
- healthStatusManager.clearStatus(TikaGrpcServer.class.getSimpleName());
- try {
- TikaGrpcServer.this.stop();
- } catch (InterruptedException e) {
- e.printStackTrace(System.err);
- }
- System.err.println("*** server shut down");
- }));
- }
-
- public void stop() throws InterruptedException {
- if (server != null) {
- server
- .shutdown()
- .awaitTermination(30, TimeUnit.SECONDS);
- }
- }
-
- /**
- * Await termination on the main thread since the grpc library uses daemon threads.
- */
- public void blockUntilShutdown() throws InterruptedException {
- if (server != null) {
- server.awaitTermination();
- }
- }
-
- /**
- * Main launches the server from the command line.
- */
- public static void main(String[] args) throws Exception {
- TikaGrpcServer server = new TikaGrpcServer();
- JCommander commander = JCommander
- .newBuilder()
- .addObject(server)
- .build();
-
- commander.parse(args);
-
- if (server.help) {
- commander.usage();
- return;
- }
-
- server.start();
- server.blockUntilShutdown();
- }
-
- public TikaGrpcServer setTikaConfigXml(File tikaConfigXml) {
- this.tikaConfigXml = tikaConfigXml;
- return this;
- }
-
- public TikaGrpcServer setServer(Server server) {
- this.server = server;
- return this;
- }
-
- public TikaGrpcServer setPort(Integer port) {
- this.port = port;
- return this;
- }
-
- public TikaGrpcServer setSecure(boolean secure) {
- this.secure = secure;
- return this;
- }
-
- public TikaGrpcServer setCertChain(File certChain) {
- this.certChain = certChain;
- return this;
- }
-
- public TikaGrpcServer setPrivateKey(File privateKey) {
- this.privateKey = privateKey;
- return this;
- }
-
- public TikaGrpcServer setPrivateKeyPassword(String privateKeyPassword) {
- this.privateKeyPassword = privateKeyPassword;
- return this;
- }
-
- public TikaGrpcServer setTrustCertCollection(File trustCertCollection) {
- this.trustCertCollection = trustCertCollection;
- return this;
- }
-
- public TikaGrpcServer setClientAuthRequired(boolean clientAuthRequired) {
- this.clientAuthRequired = clientAuthRequired;
- return this;
- }
-}
diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java
deleted file mode 100644
index 7f06cb681..000000000
--- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java
+++ /dev/null
@@ -1,420 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.grpc;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.module.jsonSchema.JsonSchema;
-import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator;
-import com.google.rpc.Status;
-import io.grpc.protobuf.StatusProto;
-import io.grpc.stub.StreamObserver;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.DeleteFetcherReply;
-import org.apache.tika.DeleteFetcherRequest;
-import org.apache.tika.FetchAndParseReply;
-import org.apache.tika.FetchAndParseRequest;
-import org.apache.tika.GetFetcherConfigJsonSchemaReply;
-import org.apache.tika.GetFetcherConfigJsonSchemaRequest;
-import org.apache.tika.GetFetcherReply;
-import org.apache.tika.GetFetcherRequest;
-import org.apache.tika.ListFetchersReply;
-import org.apache.tika.ListFetchersRequest;
-import org.apache.tika.SaveFetcherReply;
-import org.apache.tika.SaveFetcherRequest;
-import org.apache.tika.TikaGrpc;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.PipesClient;
-import org.apache.tika.pipes.core.PipesConfig;
-import org.apache.tika.pipes.core.PipesResult;
-import org.apache.tika.pipes.core.emitter.EmitKey;
-import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
-import org.apache.tika.pipes.core.fetcher.FetchKey;
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
-import org.apache.tika.pipes.core.fetcher.config.FetcherConfigContainer;
-import org.apache.tika.utils.XMLReaderUtils;
-
-class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase {
- private static final Logger LOG = LoggerFactory.getLogger(TikaGrpcServerImpl.class);
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- static {
- OBJECT_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL);
- }
- public static final JsonSchemaGenerator JSON_SCHEMA_GENERATOR = new JsonSchemaGenerator(OBJECT_MAPPER);
-
- /**
- * FetcherID is key, The pair is the Fetcher object and the Metadata
- */
- PipesConfig pipesConfig;
- PipesClient pipesClient;
- ExpiringFetcherStore expiringFetcherStore;
-
- String tikaConfigPath;
-
- TikaGrpcServerImpl(String tikaConfigPath)
- throws TikaConfigException, IOException, ParserConfigurationException,
- TransformerException, SAXException {
- File tikaConfigFile = new File(tikaConfigPath);
- if (!tikaConfigFile.canWrite()) {
- File tmpTikaConfigFile = File.createTempFile("configCopy", tikaConfigFile.getName());
- tmpTikaConfigFile.deleteOnExit();
- LOG.info("Tika config file {} is read-only. Making a temporary copy to {}", tikaConfigFile, tmpTikaConfigFile);
- String tikaConfigFileContents = FileUtils.readFileToString(tikaConfigFile, StandardCharsets.UTF_8);
- FileUtils.writeStringToFile(tmpTikaConfigFile, tikaConfigFileContents, StandardCharsets.UTF_8);
- tikaConfigFile = tmpTikaConfigFile;
- tikaConfigPath = tikaConfigFile.getAbsolutePath();
- }
- pipesConfig = PipesConfig.load(tikaConfigFile.toPath());
- pipesClient = new PipesClient(pipesConfig);
-
- expiringFetcherStore = new ExpiringFetcherStore(pipesConfig.getStaleFetcherTimeoutSeconds(),
- pipesConfig.getStaleFetcherDelaySeconds());
- this.tikaConfigPath = tikaConfigPath;
- try {
- updateTikaConfig();
- } catch (TikaException e) {
- throw new TikaConfigException("Problem updating tikaConfig", e);
- }
- }
-
- private void updateTikaConfig() throws ParserConfigurationException, IOException, SAXException, TransformerException, TikaException {
- Document tikaConfigDoc =
- DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(tikaConfigPath);
-
- Element fetchersElement = (Element) tikaConfigDoc.getElementsByTagName("fetchers").item(0);
- if (fetchersElement == null) {
- fetchersElement = tikaConfigDoc.createElement("fetchers");
- tikaConfigDoc.getDocumentElement().appendChild(fetchersElement);
- }
- for (int i = 0; i < fetchersElement.getChildNodes().getLength(); ++i) {
- fetchersElement.removeChild(fetchersElement.getChildNodes().item(i));
- }
- for (var fetcherEntry : expiringFetcherStore.getFetchers().entrySet()) {
- AbstractFetcher fetcherObject = fetcherEntry.getValue();
- Map<String, Object> fetcherConfigParams = OBJECT_MAPPER.convertValue(
- expiringFetcherStore.getFetcherConfigs().get(fetcherEntry.getKey()),
- new TypeReference<>() {
- });
- Element fetcher = tikaConfigDoc.createElement("fetcher");
- fetcher.setAttribute("class", fetcherEntry.getValue().getClass().getName());
- Element fetcherName = tikaConfigDoc.createElement("name");
- fetcherName.setTextContent(fetcherObject.getName());
- fetcher.appendChild(fetcherName);
- populateFetcherConfigs(fetcherConfigParams, tikaConfigDoc, fetcher);
- fetchersElement.appendChild(fetcher);
- }
- DOMSource source = new DOMSource(tikaConfigDoc);
- FileWriter writer = new FileWriter(tikaConfigPath, StandardCharsets.UTF_8);
- StreamResult result = new StreamResult(writer);
-
- TransformerFactory transformerFactory = XMLReaderUtils.getTransformerFactory();
- Transformer transformer = transformerFactory.newTransformer();
- transformer.transform(source, result);
- }
-
- private void populateFetcherConfigs(Map<String, Object> fetcherConfigParams,
- Document tikaConfigDoc, Element fetcher) {
- for (var configParam : fetcherConfigParams.entrySet()) {
- Element configElm = tikaConfigDoc.createElement(configParam.getKey());
- fetcher.appendChild(configElm);
- if (configParam.getValue() instanceof List) {
- List configParamVal = (List) configParam.getValue();
- String singularName = configParam.getKey().substring(0, configParam.getKey().length() - 1);
- for (Object configParamObj : configParamVal) {
- Element childElement = tikaConfigDoc.createElement(singularName);
- childElement.setTextContent(Objects.toString(configParamObj));
- configElm.appendChild(childElement);
- }
- } else {
- configElm.setTextContent(Objects.toString(configParam.getValue()));
- }
- }
- }
-
- @Override
- public void fetchAndParseServerSideStreaming(FetchAndParseRequest request,
- StreamObserver<FetchAndParseReply> responseObserver) {
- fetchAndParseImpl(request, responseObserver);
- }
-
- @Override
- public StreamObserver<FetchAndParseRequest> fetchAndParseBiDirectionalStreaming(
- StreamObserver<FetchAndParseReply> responseObserver) {
- return new StreamObserver<>() {
- @Override
- public void onNext(FetchAndParseRequest fetchAndParseRequest) {
- fetchAndParseImpl(fetchAndParseRequest, responseObserver);
- }
-
- @Override
- public void onError(Throwable throwable) {
- LOG.error("Parse error occurred", throwable);
- }
-
- @Override
- public void onCompleted() {
- responseObserver.onCompleted();
- }
- };
- }
-
- @Override
- public void fetchAndParse(FetchAndParseRequest request,
- StreamObserver<FetchAndParseReply> responseObserver) {
- fetchAndParseImpl(request, responseObserver);
- responseObserver.onCompleted();
- }
-
-
- private void fetchAndParseImpl(FetchAndParseRequest request,
- StreamObserver<FetchAndParseReply> responseObserver) {
- AbstractFetcher fetcher =
- expiringFetcherStore.getFetcherAndLogAccess(request.getFetcherId());
- if (fetcher == null) {
- throw new RuntimeException(
- "Could not find fetcher with name " + request.getFetcherId());
- }
- Metadata tikaMetadata = new Metadata();
- try {
- ParseContext parseContext = new ParseContext();
- String additionalFetchConfigJson = request.getAdditionalFetchConfigJson();
- if (StringUtils.isNotBlank(additionalFetchConfigJson)) {
- // The fetch and parse has the option to specify additional configuration
- AbstractConfig abstractConfig = expiringFetcherStore
- .getFetcherConfigs()
- .get(fetcher.getName());
- parseContext.set(FetcherConfigContainer.class, new FetcherConfigContainer()
- .setConfigClassName(abstractConfig
- .getClass().getName())
- .setJson(additionalFetchConfigJson));
- }
- PipesResult pipesResult = pipesClient.process(new FetchEmitTuple(request.getFetchKey(),
- new FetchKey(fetcher.getName(), request.getFetchKey()), new EmitKey(), tikaMetadata, parseContext, FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP));
- FetchAndParseReply.Builder fetchReplyBuilder =
- FetchAndParseReply.newBuilder()
- .setFetchKey(request.getFetchKey())
- .setStatus(pipesResult.getStatus().name());
- if (pipesResult.getStatus().equals(PipesResult.STATUS.FETCH_EXCEPTION)) {
- fetchReplyBuilder.setErrorMessage(pipesResult.getMessage());
- }
- if (pipesResult.getEmitData() != null && pipesResult.getEmitData().getMetadataList() != null) {
- for (Metadata metadata : pipesResult.getEmitData().getMetadataList()) {
- for (String name : metadata.names()) {
- String value = metadata.get(name);
- if (value != null) {
- fetchReplyBuilder.putFields(name, value);
- }
- }
- }
- }
- responseObserver.onNext(fetchReplyBuilder.build());
- } catch (IOException e) {
- throw new RuntimeException(e);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
-
- @SuppressWarnings("raw")
- @Override
- public void saveFetcher(SaveFetcherRequest request,
- StreamObserver<SaveFetcherReply> responseObserver) {
- SaveFetcherReply reply =
- SaveFetcherReply.newBuilder().setFetcherId(request.getFetcherId()).build();
- try {
- Map<String, Object> fetcherConfigMap = OBJECT_MAPPER.readValue(request.getFetcherConfigJson(), new TypeReference<>() {});
- Map<String, Param> tikaParamsMap = createTikaParamMap(fetcherConfigMap);
- saveFetcher(request.getFetcherId(), request.getFetcherClass(), fetcherConfigMap, tikaParamsMap);
- updateTikaConfig();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- responseObserver.onNext(reply);
- responseObserver.onCompleted();
- }
-
- private void saveFetcher(String name, String fetcherClassName, Map<String, Object> paramsMap, Map<String, Param> tikaParamsMap) {
- try {
- if (paramsMap == null) {
- paramsMap = new LinkedHashMap<>();
- }
- Class<? extends AbstractFetcher> fetcherClass =
- (Class<? extends AbstractFetcher>) Class.forName(fetcherClassName);
- String configClassName =
- fetcherClass.getPackageName() + ".config." + fetcherClass.getSimpleName() +
- "Config";
- Class<? extends AbstractConfig> configClass =
- (Class<? extends AbstractConfig>) Class.forName(configClassName);
- AbstractConfig configObject = OBJECT_MAPPER.convertValue(paramsMap, configClass);
- AbstractFetcher abstractFetcher =
- fetcherClass.getDeclaredConstructor(configClass).newInstance(configObject);
- abstractFetcher.setName(name);
- if (Initializable.class.isAssignableFrom(fetcherClass)) {
- Initializable initializable = (Initializable) abstractFetcher;
- initializable.initialize(tikaParamsMap);
- }
- if (expiringFetcherStore.deleteFetcher(name)) {
- LOG.info("Updating fetcher {}", name);
- } else {
- LOG.info("Creating new fetcher {}", name);
- }
- expiringFetcherStore.createFetcher(abstractFetcher, configObject);
- } catch (ClassNotFoundException | InstantiationException | IllegalAccessException |
- InvocationTargetException | NoSuchMethodException | TikaConfigException e) {
- throw new RuntimeException(e);
- }
- }
-
- private static Map<String, Param> createTikaParamMap(Map<String, Object> fetcherConfigMap) {
- Map<String, Param> tikaParamsMap = new HashMap<>();
- for (Map.Entry<String, Object> entry : fetcherConfigMap.entrySet()) {
- if (entry.getValue() != null) {
- tikaParamsMap.put(entry.getKey(), new Param<>(entry.getKey(), entry.getValue()));
- }
- }
- return tikaParamsMap;
- }
-
- static Status notFoundStatus(String fetcherId) {
- return Status.newBuilder()
- .setCode(io.grpc.Status.Code.NOT_FOUND.value())
- .setMessage("Could not find fetcher with id:" + fetcherId)
- .build();
- }
-
- @Override
- public void getFetcher(GetFetcherRequest request,
- StreamObserver<GetFetcherReply> responseObserver) {
- GetFetcherReply.Builder getFetcherReply = GetFetcherReply.newBuilder();
- AbstractConfig abstractConfig =
- expiringFetcherStore.getFetcherConfigs().get(request.getFetcherId());
- AbstractFetcher abstractFetcher = expiringFetcherStore.getFetchers().get(request.getFetcherId());
- if (abstractFetcher == null || abstractConfig == null) {
- responseObserver.onError(StatusProto.toStatusException(notFoundStatus(request.getFetcherId())));
- return;
- }
- getFetcherReply.setFetcherId(request.getFetcherId());
- getFetcherReply.setFetcherClass(abstractFetcher.getClass().getName());
- Map<String, Object> paramMap = OBJECT_MAPPER.convertValue(abstractConfig, new TypeReference<>() {});
- paramMap.forEach(
- (k, v) -> getFetcherReply.putParams(Objects.toString(k), Objects.toString(v)));
- responseObserver.onNext(getFetcherReply.build());
- responseObserver.onCompleted();
- }
-
- @Override
- public void listFetchers(ListFetchersRequest request,
- StreamObserver<ListFetchersReply> responseObserver) {
- ListFetchersReply.Builder listFetchersReplyBuilder = ListFetchersReply.newBuilder();
- for (Map.Entry<String, AbstractConfig> fetcherConfig : expiringFetcherStore.getFetcherConfigs()
- .entrySet()) {
- GetFetcherReply.Builder replyBuilder = saveFetcherReply(fetcherConfig);
- listFetchersReplyBuilder.addGetFetcherReplies(replyBuilder.build());
- }
- responseObserver.onNext(listFetchersReplyBuilder.build());
- responseObserver.onCompleted();
- }
-
- private GetFetcherReply.Builder saveFetcherReply(
- Map.Entry<String, AbstractConfig> fetcherConfig) {
- AbstractFetcher abstractFetcher =
- expiringFetcherStore.getFetchers().get(fetcherConfig.getKey());
- AbstractConfig abstractConfig =
- expiringFetcherStore.getFetcherConfigs().get(fetcherConfig.getKey());
- GetFetcherReply.Builder replyBuilder =
- GetFetcherReply.newBuilder().setFetcherClass(abstractFetcher.getClass().getName())
- .setFetcherId(abstractFetcher.getName());
- loadParamsIntoReply(abstractConfig, replyBuilder);
- return replyBuilder;
- }
-
- private static void loadParamsIntoReply(AbstractConfig abstractConfig,
- GetFetcherReply.Builder replyBuilder) {
- Map<String, Object> paramMap =
- OBJECT_MAPPER.convertValue(abstractConfig, new TypeReference<>() {
- });
- if (paramMap != null) {
- paramMap.forEach(
- (k, v) -> replyBuilder.putParams(Objects.toString(k), Objects.toString(v)));
- }
- }
-
- @Override
- public void deleteFetcher(DeleteFetcherRequest request,
- StreamObserver<DeleteFetcherReply> responseObserver) {
- boolean successfulDelete = deleteFetcher(request.getFetcherId());
- if (successfulDelete) {
- try {
- updateTikaConfig();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- responseObserver.onNext(DeleteFetcherReply.newBuilder().setSuccess(successfulDelete).build());
- responseObserver.onCompleted();
- }
-
- @Override
- public void getFetcherConfigJsonSchema(GetFetcherConfigJsonSchemaRequest request, StreamObserver<GetFetcherConfigJsonSchemaReply> responseObserver) {
- GetFetcherConfigJsonSchemaReply.Builder builder = GetFetcherConfigJsonSchemaReply.newBuilder();
- try {
- JsonSchema jsonSchema = JSON_SCHEMA_GENERATOR.generateSchema(Class.forName(request.getFetcherClass()));
- builder.setFetcherConfigJsonSchema(OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(jsonSchema));
- } catch (ClassNotFoundException | JsonProcessingException e) {
- throw new RuntimeException("Could not create json schema for " + request.getFetcherClass(), e);
- }
- responseObserver.onNext(builder.build());
- responseObserver.onCompleted();
- }
-
- private boolean deleteFetcher(String fetcherName) {
- return expiringFetcherStore.deleteFetcher(fetcherName);
- }
-}
diff --git a/tika-grpc/src/main/proto/tika.proto b/tika-grpc/src/main/proto/tika.proto
deleted file mode 100644
index 572ded7ab..000000000
--- a/tika-grpc/src/main/proto/tika.proto
+++ /dev/null
@@ -1,145 +0,0 @@
-// Copyright 2015 The gRPC Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-syntax = "proto3";
-package tika;
-
-option java_multiple_files = true;
-option java_package = "org.apache.tika";
-option java_outer_classname = "TikaProto";
-option objc_class_prefix = "HLW";
-
-// The Tika Grpc Service definition
-service Tika {
- /*
- Save a fetcher to the fetcher store.
- */
- rpc SaveFetcher(SaveFetcherRequest) returns (SaveFetcherReply) {}
- /*
- Get a fetcher's data from the fetcher store.
- */
- rpc GetFetcher(GetFetcherRequest) returns (GetFetcherReply) {}
- /*
- List fetchers that are currently in the fetcher store.
- */
- rpc ListFetchers(ListFetchersRequest) returns (ListFetchersReply) {}
- /*
- Delete a fetcher from the fetcher store.
- */
- rpc DeleteFetcher(DeleteFetcherRequest) returns (DeleteFetcherReply) {}
- /*
- Using a Fetcher in the fetcher store, send a FetchAndParse request. This will fetch, parse, and return
- the FetchParseTuple data output from Tika Pipes. This is a synchronous call that immediately returns 1 result.
- */
- rpc FetchAndParse(FetchAndParseRequest) returns (FetchAndParseReply) {}
- /*
- Using a Fetcher in the fetcher store, send a FetchAndParse request. This will fetch, parse, and return
- the FetchParseTuple data output from Tika Pipes. This will stream the data from the server in response.
- */
- rpc FetchAndParseServerSideStreaming(FetchAndParseRequest)
- returns (stream FetchAndParseReply) {}
- /*
- Using a Fetcher in the fetcher store, send a FetchAndParse request. This will fetch, parse, and return
- the FetchParseTuple data output from Tika Pipes. This serves a bi-directional stream of fetch inputs and
- parsed outputs.
- */
- rpc FetchAndParseBiDirectionalStreaming(stream FetchAndParseRequest)
- returns (stream FetchAndParseReply) {}
- /*
- Get the Fetcher Config schema for a given fetcher class.
- */
- rpc GetFetcherConfigJsonSchema(GetFetcherConfigJsonSchemaRequest) returns (GetFetcherConfigJsonSchemaReply) {}
-}
-
-message SaveFetcherRequest {
- // A unique identifier for each fetcher. If this already exists, operation will overwrite existing.
- string fetcher_id = 1;
- // The full java class name of the fetcher class. List of
- // fetcher classes is found here: https://cwiki.apache.org/confluence/display/TIKA/tika-pipes
- string fetcher_class = 2;
- // JSON string of the fetcher config object. To see the json schema from which to build this json,
- // use the GetFetcherConfigJsonSchema rpc method.
- string fetcher_config_json = 3;
-}
-
-message SaveFetcherReply {
- // The fetcher_id that was saved.
- string fetcher_id = 1;
-}
-
-message FetchAndParseRequest {
- // The ID of the fetcher in the fetcher store (previously saved by SaveFetcher) to use for the fetch.
- string fetcher_id = 1;
- // The "Fetch Key" of the item that will be fetched.
- string fetch_key = 2;
- // You can supply additional fetch configuration using this. Follows same fetch configuration json schema
- // as the fetcher configuration.
- string additional_fetch_config_json = 3;
-}
-
-message FetchAndParseReply {
- // Echoes the fetch_key that was sent in the request.
- string fetch_key = 1;
- // Metadata fields from the parse output.
- map<string, string> fields = 2;
- // The status from the message. See javadoc for org.apache.tika.pipes.PipesResult.STATUS for the list of status.
- string status = 3;
- // If there was an error, this will contain the error message.
- string error_message = 4;
-}
-
-message DeleteFetcherRequest {
- // ID of the fetcher to delete.
- string fetcher_id = 1;
-}
-
-message DeleteFetcherReply {
- // Success if the fetcher was successfully removed from the fetch store.
- bool success = 1;
-}
-
-message GetFetcherRequest {
- // ID of the fetcher for which to return config.
- string fetcher_id = 1;
-}
-
-message GetFetcherReply {
- // Echoes the ID of the fetcher being returned.
- string fetcher_id = 1;
- // The full Java class name of the Fetcher.
- string fetcher_class = 2;
- // The configuration parameters.
- map<string, string> params = 3;
-}
-
-message ListFetchersRequest {
- // List the fetchers starting at this page number
- int32 page_number = 1;
- // List this many fetchers per page.
- int32 num_fetchers_per_page = 2;
-}
-
-message ListFetchersReply {
- // List of fetcher configs returned by the Lists Fetchers service.
- repeated GetFetcherReply get_fetcher_replies = 1;
-}
-
-message GetFetcherConfigJsonSchemaRequest {
- // The full java class name of the fetcher config for which to fetch json schema.
- string fetcher_class = 1;
-}
-
-message GetFetcherConfigJsonSchemaReply {
- // The json schema that describes the fetcher config in string format.
- string fetcher_config_json_schema = 1;
-}
diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/ExpiringFetcherStoreTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/ExpiringFetcherStoreTest.java
deleted file mode 100644
index 91a8b8a19..000000000
--- a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/ExpiringFetcherStoreTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.grpc;
-
-import static org.junit.jupiter.api.Assertions.assertNull;
-
-import java.io.InputStream;
-import java.time.Duration;
-
-import org.awaitility.Awaitility;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
-
-class ExpiringFetcherStoreTest {
-
- @Test
- void createFetcher() {
- try (ExpiringFetcherStore expiringFetcherStore = new ExpiringFetcherStore(1, 5)) {
- AbstractFetcher fetcher = new AbstractFetcher() {
- @Override
- public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) {
- return null;
- }
- };
- fetcher.setName("nick");
- AbstractConfig config = new AbstractConfig() {
- };
- expiringFetcherStore.createFetcher(fetcher, config);
-
- Assertions.assertNotNull(expiringFetcherStore
- .getFetchers()
- .get(fetcher.getName()));
-
- Awaitility
- .await()
- .atMost(Duration.ofSeconds(60))
- .until(() -> expiringFetcherStore
- .getFetchers()
- .get(fetcher.getName()) == null);
-
- assertNull(expiringFetcherStore
- .getFetcherConfigs()
- .get(fetcher.getName()));
- }
- }
-}
diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java
deleted file mode 100644
index c540e6100..000000000
--- a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.grpc;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.ServerSocket;
-import java.nio.file.Paths;
-import java.time.Duration;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.ImmutableMap;
-import io.grpc.Grpc;
-import io.grpc.ManagedChannel;
-import io.grpc.TlsChannelCredentials;
-import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;
-import io.grpc.stub.StreamObserver;
-import org.apache.commons.io.FileUtils;
-import org.awaitility.Awaitility;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.handler.ResourceHandler;
-import org.eclipse.jetty.util.resource.PathResource;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.FetchAndParseReply;
-import org.apache.tika.FetchAndParseRequest;
-import org.apache.tika.SaveFetcherReply;
-import org.apache.tika.SaveFetcherRequest;
-import org.apache.tika.TikaGrpc;
-import org.apache.tika.pipes.fetcher.http.HttpFetcher;
-
-/**
- * This test will start an HTTP server using jetty.
- * Then it will start Tika Pipes Grpc service.
- * Then it will, using a bidirectional stream of data, send urls to the
- * HTTP fetcher whilst simultaneously receiving parsed output as they parse.
- */
-class PipesBiDirectionalStreamingIntegrationTest {
- static final Logger LOGGER = LoggerFactory.getLogger(PipesBiDirectionalStreamingIntegrationTest.class);
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- static File tikaConfigXmlTemplate = Paths
- .get("src", "test", "resources", "tika-pipes-test-config.xml")
- .toFile();
- static File tikaConfigXml = new File("target", "tika-config-" + UUID.randomUUID() + ".xml");
- static TikaGrpcServer grpcServer;
- static int grpcPort;
- static String httpServerUrl;
- static TikaGrpc.TikaBlockingStub tikaBlockingStub;
- static TikaGrpc.TikaStub tikaStub;
- static Server httpServer;
- static int httpServerPort;
- String httpFetcherId = "httpFetcherIdHere";
- List<String> files = Arrays.asList("014760.docx", "017091.docx", "017097.docx", "018367.docx");
-
- static int findAvailablePort() throws IOException {
- try (ServerSocket serverSocket = new ServerSocket(0)) {
- return serverSocket.getLocalPort();
- }
- }
-
- @BeforeAll
- static void setUpHttpServer() throws Exception {
- // Specify the folder from which files will be served
- httpServerPort = findAvailablePort();
- httpServer = new Server(httpServerPort);
-
- ResourceHandler resourceHandler = new ResourceHandler();
- resourceHandler.setDirAllowed(true);
- // TODO when using jetty 12:
- // resourceHandler.setBaseResourceAsString("src/test/resources/test-files")
- resourceHandler.setBaseResource(new PathResource(Paths.get("src", "test", "resources", "test-files")));
- httpServer.setHandler(resourceHandler);
- httpServer.start();
-
- httpServerUrl = "http://" + InetAddress
- .getByName("localhost")
- .getHostAddress() + ":" + httpServerPort;
- }
-
- @BeforeAll
- static void setUpGrpcServer() throws Exception {
- grpcPort = findAvailablePort();
- FileUtils.copyFile(tikaConfigXmlTemplate, tikaConfigXml);
-
- grpcServer = new TikaGrpcServer();
- grpcServer.setTikaConfigXml(tikaConfigXml);
- grpcServer.setPort(grpcPort);
- grpcServer.setSecure(true);
- grpcServer.setCertChain(Paths.get("src", "test", "resources", "certs", "server1.pem").toFile());
- grpcServer.setPrivateKey(Paths.get("src", "test", "resources", "certs", "server1.key").toFile());
- grpcServer.setTrustCertCollection(Paths.get("src", "test", "resources", "certs", "ca.pem").toFile());
- grpcServer.setClientAuthRequired(true);
- grpcServer.start();
-
- String target = InetAddress
- .getByName("localhost")
- .getHostAddress() + ":" + grpcPort;
-
- TlsChannelCredentials.Builder channelCredBuilder = TlsChannelCredentials.newBuilder();
- File clientCertChain = Paths.get("src", "test", "resources", "certs", "client.pem").toFile();
- File clientPrivateKey = Paths.get("src", "test", "resources", "certs", "client.key").toFile();
- channelCredBuilder.keyManager(clientCertChain, clientPrivateKey);
- channelCredBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE.getTrustManagers());
-
- ManagedChannel channel = Grpc
- .newChannelBuilder(target, channelCredBuilder.build())
- .build();
-
- tikaBlockingStub = TikaGrpc.newBlockingStub(channel);
- tikaStub = TikaGrpc.newStub(channel);
- }
-
- @AfterAll
- static void stopHttpServer() throws Exception {
- if (httpServer != null) {
- httpServer.stop();
- }
- }
-
- @AfterAll
- static void stopGrpcServer() throws Exception {
- if (grpcServer != null) {
- grpcServer.stop();
- }
- }
-
- @AfterAll
- static void cleanConfig() throws Exception {
- FileUtils.deleteQuietly(tikaConfigXml);
- }
-
- @BeforeEach
- void createHttpFetcher() throws Exception {
- SaveFetcherRequest saveFetcherRequest = SaveFetcherRequest
- .newBuilder()
- .setFetcherId(httpFetcherId)
- .setFetcherClass(HttpFetcher.class.getName())
- .setFetcherConfigJson(OBJECT_MAPPER.writeValueAsString(ImmutableMap
- .builder()
- .put("requestTimeout", 30_000)
- .put("socketTimeout", 30_000)
- .put("connectTimeout", 20_000)
- .put("maxConnectionsPerRoute", 200)
- .put("maxRedirects", 0)
- .put("maxSpoolSize", -1)
- .put("overallTimeout", 50_000)
- .build()))
- .build();
- SaveFetcherReply saveFetcherReply = tikaBlockingStub.saveFetcher(saveFetcherRequest);
- Assertions.assertEquals(saveFetcherReply.getFetcherId(), httpFetcherId);
- }
-
- @Test
- void testHttpFetchScenario() throws Exception {
- AtomicInteger numParsed = new AtomicInteger();
- Map<String, Map<String, String>> result = Collections.synchronizedMap(new HashMap<>());
- StreamObserver<FetchAndParseReply> responseObserver = new StreamObserver<>() {
- @Override
- public void onNext(FetchAndParseReply fetchAndParseReply) {
- LOGGER.info("Parsed: {}", fetchAndParseReply.getFetchKey());
- numParsed.incrementAndGet();
- result.put(fetchAndParseReply.getFetchKey(), fetchAndParseReply.getFieldsMap());
- }
-
- @Override
- public void onError(Throwable throwable) {
- LOGGER.error("Error occurred", throwable);
- }
-
- @Override
- public void onCompleted() {
- LOGGER.info("Completed fetching.");
- }
- };
- StreamObserver<FetchAndParseRequest> request = tikaStub.fetchAndParseBiDirectionalStreaming(responseObserver);
- for (String file : files) {
- request.onNext(FetchAndParseRequest
- .newBuilder()
- .setFetcherId(httpFetcherId)
- .setFetchKey(httpServerUrl + "/" + file)
- .build());
- }
- request.onCompleted();
-
- Awaitility.await().atMost(Duration.ofSeconds(600)).until(() -> result.size() == files.size());
-
- Assertions.assertEquals(files.size(), numParsed.get());
- }
-}
diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java
deleted file mode 100644
index a17ce1634..000000000
--- a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.grpc;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
-
-import java.io.File;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Paths;
-import java.time.Duration;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
-import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import com.asarkar.grpc.test.GrpcCleanupExtension;
-import com.asarkar.grpc.test.Resources;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.ImmutableMap;
-import io.grpc.ManagedChannel;
-import io.grpc.Server;
-import io.grpc.Status;
-import io.grpc.StatusRuntimeException;
-import io.grpc.inprocess.InProcessChannelBuilder;
-import io.grpc.inprocess.InProcessServerBuilder;
-import io.grpc.stub.StreamObserver;
-import org.apache.commons.io.FileUtils;
-import org.jetbrains.annotations.NotNull;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.DeleteFetcherReply;
-import org.apache.tika.DeleteFetcherRequest;
-import org.apache.tika.FetchAndParseReply;
-import org.apache.tika.FetchAndParseRequest;
-import org.apache.tika.GetFetcherReply;
-import org.apache.tika.GetFetcherRequest;
-import org.apache.tika.SaveFetcherReply;
-import org.apache.tika.SaveFetcherRequest;
-import org.apache.tika.TikaGrpc;
-import org.apache.tika.pipes.core.PipesResult;
-import org.apache.tika.pipes.core.fetcher.fs.FileSystemFetcher;
-
-@ExtendWith(GrpcCleanupExtension.class)
-public class TikaGrpcServerTest {
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- private static final Logger LOG = LoggerFactory.getLogger(TikaGrpcServerTest.class);
- public static final int NUM_TEST_DOCS = 2;
- static File tikaConfigXmlTemplate = Paths
- .get("src", "test", "resources", "tika-pipes-test-config.xml")
- .toFile();
- static File tikaConfigXml = new File("target", "tika-config-" + UUID.randomUUID() + ".xml");
-
-
- @BeforeAll
- static void init() throws Exception {
- FileUtils.copyFile(tikaConfigXmlTemplate, tikaConfigXml);
- }
-
- @AfterAll
- static void clean() {
- tikaConfigXml.setWritable(true);
- FileUtils.deleteQuietly(tikaConfigXml);
- }
-
- static final int NUM_FETCHERS_TO_CREATE = 10;
-
- @Test
- public void testFetcherCrud(Resources resources) throws Exception {
- Assertions.assertTrue(tikaConfigXml.setWritable(false));
- String serverName = InProcessServerBuilder.generateName();
-
- Server server = InProcessServerBuilder
- .forName(serverName)
- .directExecutor()
- .addService(new TikaGrpcServerImpl(tikaConfigXml.getAbsolutePath()))
- .build()
- .start();
- resources.register(server, Duration.ofSeconds(10));
-
- ManagedChannel channel = InProcessChannelBuilder
- .forName(serverName)
- .directExecutor()
- .build();
- resources.register(channel, Duration.ofSeconds(10));
- TikaGrpc.TikaBlockingStub blockingStub = TikaGrpc.newBlockingStub(channel);
-
- String targetFolder = new File("target").getAbsolutePath();
- // create fetchers
- for (int i = 0; i < NUM_FETCHERS_TO_CREATE; ++i) {
- String fetcherId = createFetcherId(i);
- SaveFetcherReply reply = blockingStub.saveFetcher(SaveFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .setFetcherClass(FileSystemFetcher.class.getName())
- .setFetcherConfigJson(OBJECT_MAPPER.writeValueAsString(ImmutableMap
- .builder()
- .put("basePath", targetFolder)
- .put("extractFileSystemMetadata", true)
- .build()))
- .build());
- assertEquals(fetcherId, reply.getFetcherId());
- }
- // update fetchers
- for (int i = 0; i < NUM_FETCHERS_TO_CREATE; ++i) {
- String fetcherId = createFetcherId(i);
- SaveFetcherReply reply = blockingStub.saveFetcher(SaveFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .setFetcherClass(FileSystemFetcher.class.getName())
- .setFetcherConfigJson(OBJECT_MAPPER.writeValueAsString(ImmutableMap
- .builder()
- .put("basePath", targetFolder)
- .put("extractFileSystemMetadata", false)
- .build()))
- .build());
- assertEquals(fetcherId, reply.getFetcherId());
- GetFetcherReply getFetcherReply = blockingStub.getFetcher(GetFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .build());
- assertEquals("false", getFetcherReply
- .getParamsMap()
- .get("extractFileSystemMetadata"));
- }
-
- // get fetchers
- for (int i = 0; i < NUM_FETCHERS_TO_CREATE; ++i) {
- String fetcherId = createFetcherId(i);
- GetFetcherReply getFetcherReply = blockingStub.getFetcher(GetFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .build());
- assertEquals(fetcherId, getFetcherReply.getFetcherId());
- assertEquals(FileSystemFetcher.class.getName(), getFetcherReply.getFetcherClass());
- }
-
- // delete fetchers
- for (int i = 0; i < NUM_FETCHERS_TO_CREATE; ++i) {
- String fetcherId = createFetcherId(i);
- DeleteFetcherReply deleteFetcherReply = blockingStub.deleteFetcher(DeleteFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .build());
- Assertions.assertTrue(deleteFetcherReply.getSuccess());
- StatusRuntimeException statusRuntimeException = Assertions.assertThrows(StatusRuntimeException.class, () -> blockingStub.getFetcher(GetFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .build()));
- Assertions.assertEquals(Status.NOT_FOUND
- .getCode()
- .value(), statusRuntimeException
- .getStatus()
- .getCode()
- .value());
- }
- }
-
- @NotNull
- private static String createFetcherId(int i) {
- return "nick" + i + ":is:cool:super/" + FileSystemFetcher.class;
- }
-
- @Test
- public void testBiStream(Resources resources) throws Exception {
- String serverName = InProcessServerBuilder.generateName();
-
- Server server = InProcessServerBuilder
- .forName(serverName)
- .directExecutor()
- .addService(new TikaGrpcServerImpl(tikaConfigXml.getAbsolutePath()))
- .build()
- .start();
- resources.register(server, Duration.ofSeconds(10));
-
- ManagedChannel channel = InProcessChannelBuilder
- .forName(serverName)
- .directExecutor()
- .build();
- resources.register(channel, Duration.ofSeconds(10));
- TikaGrpc.TikaBlockingStub blockingStub = TikaGrpc.newBlockingStub(channel);
- TikaGrpc.TikaStub tikaStub = TikaGrpc.newStub(channel);
-
- String fetcherId = createFetcherId(1);
- String targetFolder = new File("target").getAbsolutePath();
- SaveFetcherReply reply = blockingStub.saveFetcher(SaveFetcherRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .setFetcherClass(FileSystemFetcher.class.getName())
- .setFetcherConfigJson(OBJECT_MAPPER.writeValueAsString(ImmutableMap
- .builder()
- .put("basePath", targetFolder)
- .put("extractFileSystemMetadata", true)
- .build()))
- .build());
-
- assertEquals(fetcherId, reply.getFetcherId());
-
- List<FetchAndParseReply> successes = Collections.synchronizedList(new ArrayList<>());
- List<FetchAndParseReply> errors = Collections.synchronizedList(new ArrayList<>());
- AtomicBoolean finished = new AtomicBoolean(false);
-
- StreamObserver<FetchAndParseReply> replyStreamObserver = new StreamObserver<>() {
- @Override
- public void onNext(FetchAndParseReply fetchAndParseReply) {
- LOG.debug("Fetched {} with metadata {}", fetchAndParseReply.getFetchKey(), fetchAndParseReply.getFieldsMap());
- if (PipesResult.STATUS.FETCH_EXCEPTION.name().equals(fetchAndParseReply.getStatus())) {
- errors.add(fetchAndParseReply);
- } else {
- successes.add(fetchAndParseReply);
- }
- }
-
- @Override
- public void onError(Throwable throwable) {
- fail(throwable);
- }
-
- @Override
- public void onCompleted() {
- LOG.info("Stream completed");
- finished.set(true);
- }
- };
-
- StreamObserver<FetchAndParseRequest> requestStreamObserver = tikaStub.fetchAndParseBiDirectionalStreaming(replyStreamObserver);
-
- File testDocumentFolder = new File("target/" + DateTimeFormatter
- .ofPattern("yyyy_MM_dd_HH_mm_ssSSS", Locale.getDefault())
- .format(LocalDateTime.now(ZoneId.systemDefault())) + "-" + UUID.randomUUID());
- assertTrue(testDocumentFolder.mkdir());
- try {
- for (int i = 0; i < NUM_TEST_DOCS; ++i) {
- File testFile = new File(testDocumentFolder, "test-" + i + ".html");
- FileUtils.writeStringToFile(testFile, "<html><body>test " + i + "</body></html>", StandardCharsets.UTF_8);
- }
- File[] testDocuments = testDocumentFolder.listFiles();
- assertNotNull(testDocuments);
- for (File testDocument : testDocuments) {
- requestStreamObserver.onNext(FetchAndParseRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .setFetchKey(testDocument.getAbsolutePath())
- .build());
- }
- // Now test error condition
- requestStreamObserver.onNext(FetchAndParseRequest
- .newBuilder()
- .setFetcherId(fetcherId)
- .setFetchKey("does not exist")
- .build());
- requestStreamObserver.onCompleted();
- assertEquals(NUM_TEST_DOCS, successes.size());
- assertEquals(1, errors.size());
- assertTrue(finished.get());
- } finally {
- FileUtils.deleteDirectory(testDocumentFolder);
- }
- }
-}
diff --git a/tika-grpc/src/test/resources/certs/README.md b/tika-grpc/src/test/resources/certs/README.md
deleted file mode 100644
index 7373d5635..000000000
--- a/tika-grpc/src/test/resources/certs/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# Test certs for Tika Grpc mTLS
-
-Generate these using script as documented here:
-
-https://github.com/grpc/grpc-java/tree/b3ffb5078df361d7460786e134db7b5c00939246/testing/src/main/resources/certs
diff --git a/tika-grpc/src/test/resources/certs/ca.key b/tika-grpc/src/test/resources/certs/ca.key
deleted file mode 100644
index 03be0bfa6..000000000
--- a/tika-grpc/src/test/resources/certs/ca.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCwYvShd+UXQvOg
-z4GH6pRT3KGrPDbDw45fma7+I0LJQ4GupoeLuYYfHvcYPTV2I3MLO+VxCp00gfo1
-BIvsNOkGNxrrqNhP27ve9l7YwOuvWdVu4u9+73znRx3GJQ4ie/nF/z6xMcbQL5r5
-UC8yGwuJGOyr6VcpEnKTnORtuwRPJuqnGgn4rsKhLLfJz+RAhjdOKnAS3CQo/iHP
-KjoqIZ38M97GJ7icFQic3dtLUFR41nnN5ogLZ6DduR55btypPnlv5h6foLFjRMST
-MEroAq39ZSJqUoyBPTBtPFFk7uRQIfdKrp7/Bd4V0n4e91Us+UCDlOcxo2lF1CKH
-/ydEWmx3AgMBAAECggEAKrDosKQKKKUlvkg6+6CFIf8GiiFax+ru7KiPuCbkpT3X
-h2P67pCKq8Gc4Jr/84YE9DUdBU0iW3ESE/7ztsnflIeF1n/ZSwrN39sVfbTD1n8R
-r3LxsHFac8e8pxaU4zfKbmemztBTZFQBWFJV+fSdyCLmNX2WgPRcEuooR366PkWv
-xZLAxeDGqpnsa62o1GdGmalxx8aljLN/QcbQi73mR9Osim1OtSd1cyDlZ/8x6OoV
-Ae5GDN3Bj0hO9ZKzNWTbQpRw9SHKU6sWXtHlcDx4xi5kN/n9aptn7kixbY9y8uOM
-5zjErVGWvxdP94IvlSkrkenwnIjlHBtdlAjVuCFioQKBgQDoJLyfHNWPBnjPGVnK
-xcbIIwmf4C9UnZBbHRD3YxU/GBpsPgPh9EwhQTAXlGQGHeuslxCIh4cEfbOIrJ9b
-/s3OqeL9CSUaz/N+1av1ZuwOI9CEvNPi51IK+rXNRmVJG8pG6RaKNx57pXaFtmqq
-FUtC7twbPECvjspapn61nZYSiQKBgQDCg1tpGwZJJOCIkhYH4wFc4j4p0LxIcBJ2
-E3L9VnQ+APT/x8uitkZsuRY9tmWcHK8/zWTc1GpFdwGUJ9+Yzvprtej+P/buxM9J
-Y6ZJZdCIHWDuh3eq+sXS4lwr5fi7ir5m97npG1bXPlOoYIJ7p172EyoNmurRIgiP
-LWnzK0jG/wKBgQCRQtOouNFFcyZLaTCPutxdRddy7ESRrRq0eOax9pVH6tw12URy
-snyk3naqepdwYG6li82zsSKig8nA/0uktDeyVwoLjhpiwbc7KZc1sxaI7o4/US1B
-McBb0G/MqH0elz4myxnomP8BHhOhLflmvnZexrqCbFyJvk8PFFn7aUWMCQKBgDvX
-9BCzOszYJqh94X9NrQapqJxu1u6mZFelhjRBHARTgQ0MqC8IS0R58UjNTBeqj5Re
-mdCDHar/gSHW3qkBzPPEhMlsXol5TZjzqp5cT7sA5uicDwowmxpVgCwVVeBFQG0n
-fDAmtCIGz/A2uQ5YIRQuMzr6VZJAGUgLndQtlfd7AoGBAMq1imggFKd1rt49XCnO
-t97lpWOT+TlWYblHr01tOw+esawG5MFucqVI6tGpBSccTRQw6orWf4GK3KmkgQ6J
-UgHKjwYsA0sf4U5vppkdkbAbM/WwUPOTQpGFRERyJqMqFGIc4wMtZOJBxXwf+9iD
-l8tvan8w/6HugqnI7qqkTgLq
------END PRIVATE KEY-----
diff --git a/tika-grpc/src/test/resources/certs/ca.pem b/tika-grpc/src/test/resources/certs/ca.pem
deleted file mode 100644
index 49d39cd8e..000000000
--- a/tika-grpc/src/test/resources/certs/ca.pem
+++ /dev/null
@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDWjCCAkKgAwIBAgIUWrP0VvHcy+LP6UuYNtiL9gBhD5owDQYJKoZIhvcNAQEL
-BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
-GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTIw
-MDMxNzE4NTk1MVoXDTMwMDMxNTE4NTk1MVowVjELMAkGA1UEBhMCQVUxEzARBgNV
-BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
-ZDEPMA0GA1UEAwwGdGVzdGNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
-AQEAsGL0oXflF0LzoM+Bh+qUU9yhqzw2w8OOX5mu/iNCyUOBrqaHi7mGHx73GD01
-diNzCzvlcQqdNIH6NQSL7DTpBjca66jYT9u73vZe2MDrr1nVbuLvfu9850cdxiUO
-Inv5xf8+sTHG0C+a+VAvMhsLiRjsq+lXKRJyk5zkbbsETybqpxoJ+K7CoSy3yc/k
-QIY3TipwEtwkKP4hzyo6KiGd/DPexie4nBUInN3bS1BUeNZ5zeaIC2eg3bkeeW7c
-qT55b+Yen6CxY0TEkzBK6AKt/WUialKMgT0wbTxRZO7kUCH3Sq6e/wXeFdJ+HvdV
-LPlAg5TnMaNpRdQih/8nRFpsdwIDAQABoyAwHjAMBgNVHRMEBTADAQH/MA4GA1Ud
-DwEB/wQEAwICBDANBgkqhkiG9w0BAQsFAAOCAQEAkTrKZjBrJXHps/HrjNCFPb5a
-THuGPCSsepe1wkKdSp1h4HGRpLoCgcLysCJ5hZhRpHkRihhef+rFHEe60UePQO3S
-CVTtdJB4CYWpcNyXOdqefrbJW5QNljxgi6Fhvs7JJkBqdXIkWXtFk2eRgOIP2Eo9
-/OHQHlYnwZFrk6sp4wPyR+A95S0toZBcyDVz7u+hOW0pGK3wviOe9lvRgj/H3Pwt
-bewb0l+MhRig0/DVHamyVxrDRbqInU1/GTNCwcZkXKYFWSf92U+kIcTth24Q1gcw
-eZiLl5FfrWokUNytFElXob0V0a5/kbhiLc3yWmvWqHTpqCALbVyF+rKJo2f5Kw==
------END CERTIFICATE-----
diff --git a/tika-grpc/src/test/resources/certs/client.key b/tika-grpc/src/test/resources/certs/client.key
deleted file mode 100644
index 349b40033..000000000
--- a/tika-grpc/src/test/resources/certs/client.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCyqYRp+DXVp72N
-FbQH8hdhTZLycZXOlJhmMsrJmrjn2p7pI/8mTZ/0FC+SGWBGZV+ELiHrmCX5zfaI
-Lr9Iuw7Ghr3Vzoefi8r62rLupVPNi/qdqyjWk2dECHC9Z3+Ag3KzKTyerXWjKcvy
-KVmM0ZxE0RXhDW/RoQbqZsU2GKg1B2rhUU8KN0gVmKn0rJHOxzRVSYeYLYp5Yn7K
-rtPJcKyo9aVuEr7dGANzpyF6lg/nYBWc+9SGwkoLdFvKvABYJMyrbNhHUQfv0fza
-Z0P86dfTENrDxzALrzGnqcx3KTrwJjkZ/aSr1tyD0/tXvukRFiPxWBJhjHQ70GqT
-FQY19RbhAgMBAAECggEAIL8JUhL4awyvpWhQ8xPgTSlWwbEn8BE0TacJnCILuhNM
-BRdf8LlRk/8PKQwVpVF3TFbYSMI+U6b4hMVssfv3HVQc/083dHq+3XOwUCVlUstR
-SAzTE2E5EDMr1stdh0SQhV4Nilfos9s5Uk1Z6IGSztoz1GgOErIc/mGPy/aA/hbr
-fRWHvTp35+MbCJSvZuOeevX2iLs0dNzqdk6DiOWIH/BVGirVPtO6ykrkuTj1FWiN
-hyZ3MBChShlNH2poNX46ntOc7nEus0qteOgxBK8lummFEtlehCA7hd/8xuvYlP0k
-7aN684LCRDajmAGpoZO57NSDYQhAFGZeUZ93SMFucQKBgQDe7GGkzZFEiv91u1q9
-lgMy1h5dZjIZKgQaOarPC6wCQMUdqCf6cSLsAPr4T8EDoWsnY7dSnrTZ6YCIFL1T
-idg8M3BQXipICCJkFORS76pKKZ0wMn3/NgkSepsmNct91WHr6okvx4tOaoRCtdzU
-g7jt4Mr3sfLCiZtqTQyySdMUEwKBgQDNK+ZFKL0XhkWZP+PGKjWG8LWpPiK3d78/
-wYBFXzSTGlkr6FvRmYtZeNwXWRYLB4UxZ9At4hbJVEdi/2dITOz/sehVDyCAjjs3
-gycsc3UJqiZbcw5XKhI5TWBuWxkKENdbMSayogVbp2aSYoRblH764//t0ACmbfTW
-KUQRQPB/uwKBgQC5QjjjfPL8w4cJkGoYpFKELO2PMR7xSrmeEc6hwlFwjeNCgjy3
-JM6g0y++rIj7O2qRkY0IXFxvvF3UuWedxTCu1xC/uYHp2ti506LsScB7YZoAM/YB
-4iYn9Tx6xLoYGP0H0iGwU2SyBlNkHT8oXU+SYP5MWtYkVbeS3/VtNWz1gQKBgQCA
-6Nk4kN0mH7YxEKRzSOfyzeDF4oV7kuB2FYUbkTL+TirC3K58JiYY5Egc31trOKFm
-Jlz1xz0b6DkmKWTiV3r9OPHKJ8P7IeJxAZWmZzCdDuwkv0i+WW+z0zsIe3JjEavN
-3zb6O7R0HtziksWoqMeTqZeO+wa9iw6vVKQw1wWEqwKBgFHfahFs0DZ5cUTpGpBt
-F/AQG7ukgipB6N6AkB9kDbgCs1FLgd199MQrEncug5hfpq8QerbyMatmA+GXoGMb
-7vztKEH85yzp4n02FNL6H7xL4VVILvyZHdolmiORJ4qT2hZnl8pEQ2TYuF4RlHUd
-nSwXX+2o0J/nF85fm4AwWKAc
------END PRIVATE KEY-----
diff --git a/tika-grpc/src/test/resources/certs/client.pem b/tika-grpc/src/test/resources/certs/client.pem
deleted file mode 100644
index 8815875f3..000000000
--- a/tika-grpc/src/test/resources/certs/client.pem
+++ /dev/null
@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDNzCCAh8CFGyX00RCepOv/qCJ1oVdTtY92U83MA0GCSqGSIb3DQEBCwUAMFYx
-CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
-cm5ldCBXaWRnaXRzIFB0eSBMdGQxDzANBgNVBAMMBnRlc3RjYTAeFw0yMDAzMTgw
-MTA2MTBaFw0zMDAzMTYwMTA2MTBaMFoxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApT
-b21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEzAR
-BgNVBAMMCnRlc3RjbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
-AQCyqYRp+DXVp72NFbQH8hdhTZLycZXOlJhmMsrJmrjn2p7pI/8mTZ/0FC+SGWBG
-ZV+ELiHrmCX5zfaILr9Iuw7Ghr3Vzoefi8r62rLupVPNi/qdqyjWk2dECHC9Z3+A
-g3KzKTyerXWjKcvyKVmM0ZxE0RXhDW/RoQbqZsU2GKg1B2rhUU8KN0gVmKn0rJHO
-xzRVSYeYLYp5Yn7KrtPJcKyo9aVuEr7dGANzpyF6lg/nYBWc+9SGwkoLdFvKvABY
-JMyrbNhHUQfv0fzaZ0P86dfTENrDxzALrzGnqcx3KTrwJjkZ/aSr1tyD0/tXvukR
-FiPxWBJhjHQ70GqTFQY19RbhAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAFXCewK8
-cWT+zWxXyGFnouFSBzTi0BMBJRrhsiNoiQxkqityJHWFExiQZie+7CA+EabXCQUB
-+JwMSWM29j3mSw10DTfmC3rhheQqGxy304BZyUpdpvI2dt3p/mcsE7O+p4sQrSep
-gijiDssKAfxTAmUM93N6+Q8yJK5immxlbeYfijoBvmkzyB/B+qNRPsx0n7aFGnfv
-oWfkW296iPhWLiwknpC3xB6oK3vRbK4Zj1OaGb0grK7VN8EyhBix2xVF61i4dzCK
-kMIpl7CUpw1Mb2z8q3F2bHBS7iF7g1Ccn5VGcO+aJ+6PWydaeqJ6VEBF0Nwv9woe
-mL5AluNRLaqjZvE=
------END CERTIFICATE-----
diff --git a/tika-grpc/src/test/resources/certs/server1.key b/tika-grpc/src/test/resources/certs/server1.key
deleted file mode 100644
index 086462992..000000000
--- a/tika-grpc/src/test/resources/certs/server1.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDnE443EknxvxBq
-6+hvn/t09hl8hx366EBYvZmVM/NC+7igXRAjiJiA/mIaCvL3MS0Iz5hBLxSGICU+
-WproA3GCIFITIwcf/ETyWj/5xpgZ4AKrLrjQmmX8mhwUajfF3UvwMJrCOVqPp67t
-PtP+2kBXaqrXdvnvXR41FsIB8V7zIAuIZB6bHQhiGVlc1sgZYsE2EGG9WMmHtS86
-qkAOTjG2XyjmPTGAwhGDpYkYrpzp99IiDh4/Veai81hn0ssQkbry0XRD/Ig3jcHh
-23WiriPNJ0JsbgXUSLKRPZObA9VgOLy2aXoN84IMaeK3yy+cwSYG/99w93fUZJte
-MXwz4oYZAgMBAAECggEBAIVn2Ncai+4xbH0OLWckabwgyJ4IM9rDc0LIU368O1kU
-koais8qP9dujAWgfoh3sGh/YGgKn96VnsZjKHlyMgF+r4TaDJn3k2rlAOWcurGlj
-1qaVlsV4HiEzp7pxiDmHhWvp4672Bb6iBG+bsjCUOEk/n9o9KhZzIBluRhtxCmw5
-nw4Do7z00PTvN81260uPWSc04IrytvZUiAIx/5qxD72bij2xJ8t/I9GI8g4FtoVB
-8pB6S/hJX1PZhh9VlU6Yk+TOfOVnbebG4W5138LkB835eqk3Zz0qsbc2euoi8Hxi
-y1VGwQEmMQ63jXz4c6g+X55ifvUK9Jpn5E8pq+pMd7ECgYEA93lYq+Cr54K4ey5t
-sWMa+ye5RqxjzgXj2Kqr55jb54VWG7wp2iGbg8FMlkQwzTJwebzDyCSatguEZLuB
-gRGroRnsUOy9vBvhKPOch9bfKIl6qOgzMJB267fBVWx5ybnRbWN/I7RvMQf3k+9y
-biCIVnxDLEEYyx7z85/5qxsXg/MCgYEA7wmWKtCTn032Hy9P8OL49T0X6Z8FlkDC
-Rk42ygrc/MUbugq9RGUxcCxoImOG9JXUpEtUe31YDm2j+/nbvrjl6/bP2qWs0V7l
-dTJl6dABP51pCw8+l4cWgBBX08Lkeen812AAFNrjmDCjX6rHjWHLJcpS18fnRRkP
-V1d/AHWX7MMCgYEA6Gsw2guhp0Zf2GCcaNK5DlQab8OL4Hwrpttzo4kuTlwtqNKp
-Q9H4al9qfF4Cr1TFya98+EVYf8yFRM3NLNjZpe3gwYf2EerlJj7VLcahw0KKzoN1
-QBENfwgPLRk5sDkx9VhSmcfl/diLroZdpAwtv3vo4nEoxeuGFbKTGx3Qkf0CgYEA
-xyR+dcb05Ygm3w4klHQTowQ10s1H80iaUcZBgQuR1ghEtDbUPZHsoR5t1xCB02ys
-DgAwLv1bChIvxvH/L6KM8ovZ2LekBX4AviWxoBxJnfz/EVau98B0b1auRN6eSC83
-FRuGldlSOW1z/nSh8ViizSYE5H5HX1qkXEippvFRE88CgYB3Bfu3YQY60ITWIShv
-nNkdcbTT9eoP9suaRJjw92Ln+7ZpALYlQMKUZmJ/5uBmLs4RFwUTQruLOPL4yLTH
-awADWUzs3IRr1fwn9E+zM8JVyKCnUEM3w4N5UZskGO2klashAd30hWO+knRv/y0r
-uGIYs9Ek7YXlXIRVrzMwcsrt1w==
------END PRIVATE KEY-----
diff --git a/tika-grpc/src/test/resources/certs/server1.pem b/tika-grpc/src/test/resources/certs/server1.pem
deleted file mode 100644
index 88244f856..000000000
--- a/tika-grpc/src/test/resources/certs/server1.pem
+++ /dev/null
@@ -1,22 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDtDCCApygAwIBAgIUbJfTREJ6k6/+oInWhV1O1j3ZT0IwDQYJKoZIhvcNAQEL
-BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
-GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTIw
-MDMxODAzMTA0MloXDTMwMDMxNjAzMTA0MlowZTELMAkGA1UEBhMCVVMxETAPBgNV
-BAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRUwEwYDVQQKDAxFeGFtcGxl
-LCBDby4xGjAYBgNVBAMMESoudGVzdC5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0B
-AQEFAAOCAQ8AMIIBCgKCAQEA5xOONxJJ8b8Qauvob5/7dPYZfIcd+uhAWL2ZlTPz
-Qvu4oF0QI4iYgP5iGgry9zEtCM+YQS8UhiAlPlqa6ANxgiBSEyMHH/xE8lo/+caY
-GeACqy640Jpl/JocFGo3xd1L8DCawjlaj6eu7T7T/tpAV2qq13b5710eNRbCAfFe
-8yALiGQemx0IYhlZXNbIGWLBNhBhvVjJh7UvOqpADk4xtl8o5j0xgMIRg6WJGK6c
-6ffSIg4eP1XmovNYZ9LLEJG68tF0Q/yIN43B4dt1oq4jzSdCbG4F1EiykT2TmwPV
-YDi8tml6DfOCDGnit8svnMEmBv/fcPd31GSbXjF8M+KGGQIDAQABo2swaTAJBgNV
-HRMEAjAAMAsGA1UdDwQEAwIF4DBPBgNVHREESDBGghAqLnRlc3QuZ29vZ2xlLmZy
-ghh3YXRlcnpvb2kudGVzdC5nb29nbGUuYmWCEioudGVzdC55b3V0dWJlLmNvbYcE
-wKgBAzANBgkqhkiG9w0BAQsFAAOCAQEAS8hDQA8PSgipgAml7Q3/djwQ644ghWQv
-C2Kb+r30RCY1EyKNhnQnIIh/OUbBZvh0M0iYsy6xqXgfDhCB93AA6j0i5cS8fkhH
-Jl4RK0tSkGQ3YNY4NzXwQP/vmUgfkw8VBAZ4Y4GKxppdATjffIW+srbAmdDruIRM
-wPeikgOoRrXf0LA1fi4TqxARzeRwenQpayNfGHTvVF9aJkl8HoaMunTAdG5pIVcr
-9GKi/gEMpXUJbbVv3U5frX1Wo4CFo+rZWJ/LyCMeb0jciNLxSdMwj/E/ZuExlyeZ
-gc9ctPjSMvgSyXEKv6Vwobleeg88V2ZgzenziORoWj4KszG/lbQZvg==
------END CERTIFICATE-----
diff --git a/tika-grpc/src/test/resources/tika-pipes-test-config.xml b/tika-grpc/src/test/resources/tika-pipes-test-config.xml
deleted file mode 100644
index e4006edb3..000000000
--- a/tika-grpc/src/test/resources/tika-pipes-test-config.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?><!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<properties>
- <async>
- <staleFetcherTimeoutSeconds>600</staleFetcherTimeoutSeconds>
- <staleFetcherDelaySeconds>60</staleFetcherDelaySeconds>
- </async>
- <pipes>
- <params>
- <numClients>2</numClients>
- <forkedJvmArgs>
- <arg>-Xmx1g</arg>
- <arg>-XX:ParallelGCThreads=2</arg>
- </forkedJvmArgs>
- <timeoutMillis>60000</timeoutMillis>
- <maxForEmitBatchBytes>-1</maxForEmitBatchBytes> <!-- disable emit -->
- </params>
- </pipes>
- <fetchers>
- </fetchers>
-</properties>
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index 77baf60cb..e3a0e9d7a 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -439,6 +439,7 @@
<snappy.version>1.1.10.8</snappy.version>
<solrj.version>9.9.0</solrj.version>
<spring.version>6.2.11</spring.version>
+ <spring.boot.version>3.4.1</spring.boot.version>
<sqlite.version>3.50.3.0</sqlite.version>
<stax.ex.version>2.1.0</stax.ex.version>
<testcontainers.version>1.21.3</testcontainers.version>
@@ -456,6 +457,8 @@
<zstd.version>1.5.7-4</zstd.version>
<nimbus-jose-jwt.version>10.5</nimbus-jose-jwt.version>
<javacpp.version>1.5.12</javacpp.version>
+ <pf4j.version>3.12.0</pf4j.version>
+ <lombok.version>1.18.36</lombok.version>
<!-- needs to exist even if empty due to problems with jacoco-maven-plugin -->
<addmod />
@@ -479,6 +482,13 @@
<version>2.9.0</version>
</dependency>
<!-- boms -->
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-dependencies</artifactId>
+ <version>${spring.boot.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
<dependency>
<groupId>com.fasterxml.jackson</groupId>
<artifactId>jackson-bom</artifactId>
@@ -1149,6 +1159,16 @@
<artifactId>nimbus-jose-jwt</artifactId>
<version>${nimbus-jose-jwt.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.pf4j</groupId>
+ <artifactId>pf4j</artifactId>
+ <version>${pf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>${lombok.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/tika-pipes/pom.xml b/tika-pipes/pom.xml
index 204311932..cc06d9615 100644
--- a/tika-pipes/pom.xml
+++ b/tika-pipes/pom.xml
@@ -1,19 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
-->
+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -25,69 +30,17 @@
</parent>
<artifactId>tika-pipes</artifactId>
- <name>Apache Tika pipes</name>
- <url>https://tika.apache.org/</url>
<packaging>pom</packaging>
+ <name>Apache Tika Pipes</name>
+ <description>Apache Tika Pipes aggregator project</description>
<modules>
<module>tika-pipes-core</module>
- <module>tika-httpclient-commons</module>
- <module>tika-fetchers</module>
- <module>tika-emitters</module>
- <module>tika-pipes-iterators</module>
- <module>tika-pipes-reporters</module>
- <module>tika-async-cli</module>
+ <module>tika-pipes-pipe-iterators</module>
+ <module>tika-pipes-fetchers</module>
+ <module>tika-pipes-emitters</module>
+ <module>tika-pipes-proto</module>
+ <module>tika-pipes-grpc</module>
+ <module>tika-pipes-cli</module>
</modules>
- <dependencies>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <version>${checkstyle.plugin.version}</version>
- <dependencies>
- <dependency>
- <groupId>com.puppycrawl.tools</groupId>
- <artifactId>checkstyle</artifactId>
- <version>${puppycrawl.version}</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>validate</id>
- <phase>validate</phase>
- <configuration>
- <configLocation>checkstyle.xml</configLocation>
- <inputEncoding>UTF-8</inputEncoding>
- <consoleOutput>true</consoleOutput>
- <logViolationsToConsole>true</logViolationsToConsole>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <testSourceDirectories>${project.basedir}/src/test/java</testSourceDirectories>
- <violationSeverity>error</violationSeverity>
- <failOnViolation>true</failOnViolation>
- <excludeGeneratedSources>true</excludeGeneratedSources>
- </configuration>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
+</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-async-cli/pom.xml b/tika-pipes/tika-async-cli/pom.xml
deleted file mode 100644
index 47991bcc3..000000000
--- a/tika-pipes/tika-async-cli/pom.xml
+++ /dev/null
@@ -1,134 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-pipes</artifactId>
- <version>4.0.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-async-cli</artifactId>
-
- <name>Apache Tika Async CLI</name>
- <url>https://tika.apache.org/</url>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
- <!-- logging -->
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-serialization</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.reporters.fs.status</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.tika.async.cli.TikaAsyncCLI</mainClass>
- <manifestEntries>
- <Multi-Release>true</Multi-Release>
- </manifestEntries>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
diff --git a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/SimpleAsyncConfig.java b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/SimpleAsyncConfig.java
deleted file mode 100644
index 34458beb0..000000000
--- a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/SimpleAsyncConfig.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.async.cli;
-
-class SimpleAsyncConfig {
-
- private String inputDir;
- private String outputDir;
- private Integer numClients;
- private Long timeoutMs;
- private String xmx;
- private String fileList;
-
- public SimpleAsyncConfig(String inputDir, String outputDir, Integer numClients, Long timeoutMs, String xmx, String fileList) {
- this.inputDir = inputDir;
- this.outputDir = outputDir;
- this.numClients = numClients;
- this.timeoutMs = timeoutMs;
- this.xmx = xmx;
- this.fileList = fileList;
- }
-
- public String getInputDir() {
- return inputDir;
- }
-
- public String getOutputDir() {
- return outputDir;
- }
-
- public Integer getNumClients() {
- return numClients;
- }
-
- public Long getTimeoutMs() {
- return timeoutMs;
- }
-
- public String getXmx() {
- return xmx;
- }
-
- public String getFileList() {
- return fileList;
- }
-}
diff --git a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java
deleted file mode 100644
index 769265ed1..000000000
--- a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.async.cli;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.concurrent.TimeoutException;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.async.AsyncProcessor;
-import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
-
-public class TikaAsyncCLI {
-
- private static final long TIMEOUT_MS = 600_000;
- private static final Logger LOG = LoggerFactory.getLogger(TikaAsyncCLI.class);
-
- private static Options getOptions() {
- Options options = new Options();
- options.addOption("i", "inputDir", true, "input directory");
- options.addOption("o", "outputDir", true, "output directory");
-
- options.addOption("n", "numClients", true, "number of forked clients");
- options.addOption("x", "Xmx", true, "heap for the forked clients in usual jvm heap amount, e.g. -x 1g");
- options.addOption("?", "help", false, "this help message");
- options.addOption("t", "timeoutMs", true, "timeout for each parse in milliseconds");
- options.addOption("l", "fileList", true, "file list");
-
- return options;
- }
-
- public static void main(String[] args) throws Exception {
- if (args.length == 0) {
- usage(getOptions());
- } else if (args.length == 1) {
- processWithTikaConfig(Paths.get(args[0]));
- } else {
- processCommandLine(args);
- }
- }
-
- private static void processCommandLine(String[] args) throws Exception {
-
- SimpleAsyncConfig simpleAsyncConfig = parseCommandLine(args);
-
- Path tikaConfig = null;
- try {
- tikaConfig = Files.createTempFile("tika-async-tmp-", ".xml");
- TikaConfigAsyncWriter tikaConfigAsyncWriter = new TikaConfigAsyncWriter(simpleAsyncConfig);
- tikaConfigAsyncWriter.write(tikaConfig);
- processWithTikaConfig(tikaConfig);
- } finally {
- if (tikaConfig != null) {
- Files.delete(tikaConfig);
- }
- }
- }
-
- //not private for testing purposes
- static SimpleAsyncConfig parseCommandLine(String[] args) throws ParseException, IOException {
- if (args.length == 2 && ! args[0].startsWith("-")) {
- return new SimpleAsyncConfig(args[0], args[1], null, null, null, null);
- }
-
- Options options = getOptions();
-
- CommandLineParser cliParser = new DefaultParser();
-
- CommandLine line = cliParser.parse(options, args);
- if (line.hasOption("help")) {
- usage(options);
- }
- String inputDir = null;
- String outputDir = null;
- String xmx = null;
- Long timeoutMs = null;
- Integer numClients = null;
- String fileList = null;
- if (line.hasOption("i")) {
- inputDir = line.getOptionValue("i");
- }
- if (line.hasOption("o")) {
- outputDir = line.getOptionValue("o");
- }
- if (line.hasOption("x")) {
- xmx = line.getOptionValue("x");
- }
- if (line.hasOption("t")) {
- timeoutMs = Long.parseLong(line.getOptionValue("t"));
- }
- if (line.hasOption("n")) {
- numClients = Integer.parseInt(line.getOptionValue("n"));
- }
- if (line.hasOption("l")) {
- fileList = line.getOptionValue("l");
- }
- return new SimpleAsyncConfig(inputDir, outputDir,
- numClients, timeoutMs, xmx, fileList);
- }
-
-
- private static void processWithTikaConfig(Path tikaConfigPath) throws Exception {
- PipesIterator pipesIterator = PipesIterator.build(tikaConfigPath);
- long start = System.currentTimeMillis();
- try (AsyncProcessor processor = new AsyncProcessor(tikaConfigPath, pipesIterator)) {
-
- for (FetchEmitTuple t : pipesIterator) {
- boolean offered = processor.offer(t, TIMEOUT_MS);
- if (!offered) {
- throw new TimeoutException("timed out waiting to add a fetch emit tuple");
- }
- }
- processor.finished();
- while (true) {
- if (processor.checkActive()) {
- Thread.sleep(500);
- } else {
- break;
- }
- }
- long elapsed = System.currentTimeMillis() - start;
- LOG.info("Successfully finished processing {} files in {} ms", processor.getTotalProcessed(), elapsed);
-
- }
- }
-
- private static void usage(Options options) throws IOException {
- System.out.println("Two primary options:");
- System.out.println("\t1. Specify a tika-config.xml on the commandline that includes the definitions for async");
- System.out.println("\t2. Commandline:");
- org.apache.commons.cli.help.HelpFormatter helpFormatter = org.apache.commons.cli.help.HelpFormatter.builder().get();
- helpFormatter.printHelp("tikaAsynCli", null, options, null, true);
- System.exit(1);
- }
-}
diff --git a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaConfigAsyncWriter.java b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaConfigAsyncWriter.java
deleted file mode 100644
index d24141023..000000000
--- a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaConfigAsyncWriter.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.async.cli;
-
-import java.io.IOException;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-
-import org.apache.tika.utils.StringUtils;
-
-class TikaConfigAsyncWriter {
-
- private static final String FETCHER_NAME = "fsf";
- private static final String EMITTER_NAME = "fse";
-
- private final SimpleAsyncConfig simpleAsyncConfig;
-
- TikaConfigAsyncWriter(SimpleAsyncConfig simpleAsyncConfig) {
- this.simpleAsyncConfig = simpleAsyncConfig;
- }
-
- void write(Path output) throws IOException {
- try {
- _write(output);
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- void _write(Path output) throws ParserConfigurationException, TransformerException, IOException {
- DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- Document document = dbf.newDocumentBuilder().newDocument();
- Element properties = document.createElement("properties");
- document.appendChild(properties);
- writePipesIterator(document, properties);
- writeFetchers(document, properties);
- writeEmitters(document, properties);
- writeAsync(document, properties);
- Transformer transformer = TransformerFactory
- .newInstance().newTransformer();
- transformer.setOutputProperty(OutputKeys.INDENT, "yes");
- transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
- try (Writer writer = Files.newBufferedWriter(output, StandardCharsets.UTF_8)) {
- StreamResult result = new StreamResult(writer);
- DOMSource source = new DOMSource(document);
- transformer.transform(source, result);
- }
-
- }
-
- private void writePipesIterator(Document document, Element properties) {
- if (! StringUtils.isBlank(simpleAsyncConfig.getFileList())) {
- writeFileListIterator(document, properties);
- } else {
- writeFileSystemIterator(document, properties);
- }
- }
-
- private void writeFileSystemIterator(Document document, Element properties) {
- Element pipesIterator = createAndGetElement(document, properties, "pipesIterator",
- "class", "org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator");
- appendTextElement(document, pipesIterator, "basePath",
- Paths.get(simpleAsyncConfig.getInputDir()).toAbsolutePath().toString());
- appendTextElement(document, pipesIterator, "fetcherName", FETCHER_NAME);
- appendTextElement(document, pipesIterator, "emitterName", EMITTER_NAME);
- }
-
- private void writeFileListIterator(Document document, Element properties) {
- Element pipesIterator = createAndGetElement(document, properties, "pipesIterator",
- "class", "org.apache.tika.pipes.pipesiterator.filelist.FileListPipesIterator");
- appendTextElement(document, pipesIterator, "fetcherName", FETCHER_NAME);
- appendTextElement(document, pipesIterator, "emitterName", EMITTER_NAME);
- appendTextElement(document, pipesIterator, "fileList",
- Paths.get(simpleAsyncConfig.getFileList()).toAbsolutePath().toString());
- appendTextElement(document, pipesIterator, "hasHeader", "false");
- }
-
- private void writeEmitters(Document document, Element properties) {
- Element emitters = createAndGetElement(document, properties, "emitters");
- Element emitter = createAndGetElement( document, emitters, "emitter",
- "class", "org.apache.tika.pipes.core.emitter.fs.FileSystemEmitter");
- appendTextElement(document, emitter, "name", EMITTER_NAME);
- appendTextElement(document, emitter, "basePath",
- Paths.get(simpleAsyncConfig.getOutputDir()).toAbsolutePath().toString());
- }
-
- private void writeFetchers(Document document, Element properties) {
- Element fetchers = createAndGetElement(document, properties, "fetchers");
- Element fetcher = createAndGetElement(document, fetchers, "fetcher",
- "class", "org.apache.tika.pipes.fetcher.fs.FileSystemFetcher");
- appendTextElement(document, fetcher, "name", FETCHER_NAME);
- if (!StringUtils.isBlank(simpleAsyncConfig.getInputDir())) {
- appendTextElement(document, fetcher, "basePath", Paths
- .get(simpleAsyncConfig.getInputDir())
- .toAbsolutePath()
- .toString());
- } else {
- appendTextElement(document, fetcher, "basePath", "");
- }
- }
-
- private void writeAsync(Document document, Element properties) {
- Element async = createAndGetElement(document, properties, "async");
- properties.appendChild(async);
- if (simpleAsyncConfig.getNumClients() != null) {
- appendTextElement(document, async, "numClients", Integer.toString(simpleAsyncConfig.getNumClients()));
- }
- if (simpleAsyncConfig.getXmx() != null) {
- Element forkedJvmArgs = createAndGetElement(document, async, "forkedJvmArgs");
- appendTextElement(document, forkedJvmArgs, "arg", "-Xmx" + simpleAsyncConfig.getXmx());
- }
- if (simpleAsyncConfig.getTimeoutMs() != null) {
- appendTextElement(document, async, "timeoutMillis", Long.toString(simpleAsyncConfig.getTimeoutMs()));
- }
- }
-
- private static void appendTextElement(Document document, Element parent, String itemName, String text, String... attrs) {
- Element el = createAndGetElement(document, parent, itemName, attrs);
- el.setTextContent(text);
- }
-
- private static Element createAndGetElement(Document document, Element parent, String elementName, String... attrs) {
- Element el = document.createElement(elementName);
- parent.appendChild(el);
- for (int i = 0; i < attrs.length; i += 2) {
- el.setAttribute(attrs[i], attrs[i + 1]);
- }
- return el;
- }
-
-}
diff --git a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java
deleted file mode 100644
index 4e38aac9c..000000000
--- a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.async.cli;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNull;
-
-import org.junit.jupiter.api.Test;
-
-public class AsyncCliParserTest {
-
- @Test
- public void testBasic() throws Exception {
- SimpleAsyncConfig simpleAsyncConfig = TikaAsyncCLI.parseCommandLine(new String[]{"input", "output"});
- assertEquals("input", simpleAsyncConfig.getInputDir());
- assertEquals("output", simpleAsyncConfig.getOutputDir());
- assertNull(simpleAsyncConfig.getFileList());
- assertNull(simpleAsyncConfig.getNumClients());
- assertNull(simpleAsyncConfig.getTimeoutMs());
- assertNull(simpleAsyncConfig.getXmx());
-
- simpleAsyncConfig = TikaAsyncCLI.parseCommandLine(new String[]{"-o", "output", "-i", "input"});
- assertEquals("input", simpleAsyncConfig.getInputDir());
- assertEquals("output", simpleAsyncConfig.getOutputDir());
- assertNull(simpleAsyncConfig.getFileList());
- assertNull(simpleAsyncConfig.getNumClients());
- assertNull(simpleAsyncConfig.getTimeoutMs());
- assertNull(simpleAsyncConfig.getXmx());
-
- simpleAsyncConfig = TikaAsyncCLI.parseCommandLine(new String[]{"-output", "output", "-input", "input"});
- assertEquals("input", simpleAsyncConfig.getInputDir());
- assertEquals("output", simpleAsyncConfig.getOutputDir());
- assertNull(simpleAsyncConfig.getFileList());
- assertNull(simpleAsyncConfig.getNumClients());
- assertNull(simpleAsyncConfig.getTimeoutMs());
- assertNull(simpleAsyncConfig.getXmx());
-
- simpleAsyncConfig = TikaAsyncCLI.parseCommandLine(new String[]{"--output", "output", "--input", "input"});
- assertEquals("input", simpleAsyncConfig.getInputDir());
- assertEquals("output", simpleAsyncConfig.getOutputDir());
- assertNull(simpleAsyncConfig.getFileList());
- assertNull(simpleAsyncConfig.getNumClients());
- assertNull(simpleAsyncConfig.getTimeoutMs());
- assertNull(simpleAsyncConfig.getXmx());
-
- simpleAsyncConfig = TikaAsyncCLI.parseCommandLine(new String[]{"--output=output", "--input=input"});
- assertEquals("input", simpleAsyncConfig.getInputDir());
- assertEquals("output", simpleAsyncConfig.getOutputDir());
- assertNull(simpleAsyncConfig.getFileList());
- assertNull(simpleAsyncConfig.getNumClients());
- assertNull(simpleAsyncConfig.getTimeoutMs());
- assertNull(simpleAsyncConfig.getXmx());
- }
-
- @Test
- public void testAll() throws Exception {
- SimpleAsyncConfig simpleAsyncConfig = TikaAsyncCLI.parseCommandLine(
- new String[]{"-i", "input", "-o", "output", "-n", "5", "-t", "30000", "-x", "1g"});
- assertEquals("input", simpleAsyncConfig.getInputDir());
- assertEquals("output", simpleAsyncConfig.getOutputDir());
- assertNull(simpleAsyncConfig.getFileList());
- assertEquals(5, simpleAsyncConfig.getNumClients());
- assertEquals(30000L, simpleAsyncConfig.getTimeoutMs());
- assertEquals("1g", simpleAsyncConfig.getXmx());
- }
-
- //TODO -- test for file list with and without inputDir
-}
diff --git a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncProcessorTest.java b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncProcessorTest.java
deleted file mode 100644
index 482e20ea2..000000000
--- a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncProcessorTest.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.async.cli;
-
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.List;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-import org.apache.tika.TikaTest;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.HandlerConfig;
-import org.apache.tika.pipes.core.async.AsyncProcessor;
-import org.apache.tika.pipes.core.emitter.EmitKey;
-import org.apache.tika.pipes.core.extractor.EmbeddedDocumentBytesConfig;
-import org.apache.tika.pipes.core.fetcher.FetchKey;
-import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
-import org.apache.tika.serialization.JsonMetadataList;
-
-/**
- * This should be in tika-core, but we want to avoid a dependency mess with tika-serialization
- */
-public class AsyncProcessorTest extends TikaTest {
- //TODO -- integrate json pipes iterator and run with AyncProcessor.main
- @TempDir
- private Path basedir;
- private Path inputDir;
-
- private Path bytesDir;
-
- private Path jsonDir;
-
- private Path configDir;
-
- @BeforeEach
- public void setUp() throws IOException {
- inputDir = basedir.resolve("input");
-
- bytesDir = basedir.resolve("bytes");
-
- jsonDir = basedir.resolve("json");
-
- configDir = basedir.resolve("config");
- Path tikaConfig = configDir.resolve("tika-config.xml");
-
- Files.createDirectories(basedir);
- Files.createDirectories(configDir);
- Files.createDirectories(inputDir);
-
- String xml = IOUtils.toString(AsyncProcessorTest.class.getResourceAsStream("/configs/TIKA-4207-emitter.xml"), StandardCharsets.UTF_8);
- //do stuff to xml
- xml = xml.replace("BASE_PATH", inputDir
- .toAbsolutePath()
- .toString());
- xml = xml.replace("JSON_PATH", jsonDir
- .toAbsolutePath()
- .toString());
- xml = xml.replace("BYTES_PATH", bytesDir
- .toAbsolutePath()
- .toString());
-
- Files.writeString(tikaConfig, xml, StandardCharsets.UTF_8);
-
- Path mock = inputDir.resolve("mock.xml");
- try (OutputStream os = Files.newOutputStream(mock)) {
- IOUtils.copy(getClass().getResourceAsStream("/test-documents/basic_embedded.xml"), os);
- }
- }
-
- @Test
- public void testBasic() throws Exception {
-// TikaAsyncCLI cli = new TikaAsyncCLI();
- // cli.main(new String[]{ configDir.resolve("tika-config.xml").toAbsolutePath().toString()});
- AsyncProcessor processor = new AsyncProcessor(configDir.resolve("tika-config.xml"));
-
- EmbeddedDocumentBytesConfig embeddedDocumentBytesConfig = new EmbeddedDocumentBytesConfig(true);
- embeddedDocumentBytesConfig.setIncludeOriginal(true);
- embeddedDocumentBytesConfig.setEmitter("bytes");
- embeddedDocumentBytesConfig.setSuffixStrategy(EmbeddedDocumentBytesConfig.SUFFIX_STRATEGY.NONE);
- embeddedDocumentBytesConfig.setEmbeddedIdPrefix("-");
- ParseContext parseContext = new ParseContext();
- parseContext.set(HandlerConfig.class, HandlerConfig.DEFAULT_HANDLER_CONFIG);
- parseContext.set(EmbeddedDocumentBytesConfig.class, embeddedDocumentBytesConfig);
- FetchEmitTuple t =
- new FetchEmitTuple("myId-1", new FetchKey("fs", "mock.xml"), new EmitKey("json", "emit-1"), new Metadata(), parseContext, FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT);
-
- processor.offer(t, 1000);
-
- for (int i = 0; i < 10; i++) {
- processor.offer(PipesIterator.COMPLETED_SEMAPHORE, 1000);
- }
- //TODO clean this up
- while (processor.checkActive()) {
- Thread.sleep(100);
- }
- processor.close();
-
- String container = Files.readString(bytesDir.resolve("emit-1/emit-1-0"));
- assertContains("\"dc:creator\">Nikolai Lobachevsky", container);
-
- String xmlEmbedded = Files.readString(bytesDir.resolve("emit-1/emit-1-1"));
- assertContains("name=\"dc:creator\"", xmlEmbedded);
- assertContains(">embeddedAuthor</metadata>", xmlEmbedded);
-
- List<Metadata> metadataList;
- try (BufferedReader reader = Files.newBufferedReader(jsonDir.resolve("emit-1.json"))) {
- metadataList = JsonMetadataList.fromJson(reader);
- }
- assertEquals(2, metadataList.size());
- assertContains("main_content", metadataList
- .get(0)
- .get(TikaCoreProperties.TIKA_CONTENT));
- assertContains("some_embedded_content", metadataList
- .get(1)
- .get(TikaCoreProperties.TIKA_CONTENT));
- }
-}
diff --git a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaAsyncCLITest.java b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaAsyncCLITest.java
deleted file mode 100644
index 2fd9818f2..000000000
--- a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/TikaAsyncCLITest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.async.cli;
-
-import static org.junit.jupiter.api.Assertions.assertThrows;
-
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.exception.TikaConfigException;
-
-public class TikaAsyncCLITest {
- @Test
- public void testCrash() throws Exception {
- Path config = getPath("/configs/tika-config-broken.xml");
- assertThrows(TikaConfigException.class, () -> TikaAsyncCLI.main(new String[]{config.toAbsolutePath().toString()}));
- }
-
- private Path getPath(String file) throws Exception {
- return Paths.get(this
- .getClass()
- .getResource(file)
- .toURI());
- }
-}
diff --git a/tika-pipes/tika-async-cli/src/test/resources/configs/TIKA-4207-emitter.xml b/tika-pipes/tika-async-cli/src/test/resources/configs/TIKA-4207-emitter.xml
deleted file mode 100644
index bb8bfe77f..000000000
--- a/tika-pipes/tika-async-cli/src/test/resources/configs/TIKA-4207-emitter.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<properties>
- <fetchers>
- <fetcher class="org.apache.tika.pipes.core.fetcher.fs.FileSystemFetcher">
- <name>fs</name>
- <basePath>BASE_PATH</basePath>
- </fetcher>
- </fetchers>
- <emitters>
- <emitter class="org.apache.tika.pipes.core.emitter.fs.FileSystemEmitter">
- <name>json</name>
- <basePath>JSON_PATH</basePath>
- </emitter>
- <emitter class="org.apache.tika.pipes.core.emitter.fs.FileSystemEmitter">
- <name>bytes</name>
- <basePath>BYTES_PATH</basePath>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-async-cli/src/test/resources/configs/tika-config-broken.xml b/tika-pipes/tika-async-cli/src/test/resources/configs/tika-config-broken.xml
deleted file mode 100644
index 5ee379e6f..000000000
--- a/tika-pipes/tika-async-cli/src/test/resources/configs/tika-config-broken.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <fetchers>
- <fetcher class="org.apache.tika.pipes.fetcher.s3.S3Fetcher">
- <name>s3</name>
- <region>us-east-1</region>
- <profile><!-- fill in here --></profile>
- </fetcher>
- </fetchers>
- <pipesIterator class="org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator">
- <fetcherName>fs</fetcherName>
- <basePath>basePath</basePath>
- </pipesIterator>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-async-cli/src/test/resources/test-documents/basic_embedded.xml b/tika-pipes/tika-async-cli/src/test/resources/test-documents/basic_embedded.xml
deleted file mode 100644
index 7536a1603..000000000
--- a/tika-pipes/tika-async-cli/src/test/resources/test-documents/basic_embedded.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<mock>
-
- <metadata action="add" name="dc:creator">Nikolai Lobachevsky</metadata>
- <write element="p">main_content</write>
- <!-- auto detection wasn't working for some reason; add content-type as
- is to trigger mock on the embedded -->
- <embedded filename="embed1.xml" content-type="application/mock+xml">
- <mock>
- <metadata action="add" name="dc:creator">embeddedAuthor</metadata>
- <write element="p">some_embedded_content</write>
- </mock>
- </embedded>
-
-</mock>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/pom.xml b/tika-pipes/tika-emitters/pom.xml
deleted file mode 100644
index b63c797e1..000000000
--- a/tika-pipes/tika-emitters/pom.xml
+++ /dev/null
@@ -1,47 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-pipes</artifactId>
- <version>4.0.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitters</artifactId>
- <name>Apache Tika emitters</name>
- <url>https://tika.apache.org/</url>
- <packaging>pom</packaging>
-
- <modules>
- <module>tika-emitter-s3</module>
- <module>tika-emitter-kafka</module>
- <module>tika-emitter-solr</module>
- <module>tika-emitter-opensearch</module>
- <module>tika-emitter-gcs</module>
- <module>tika-emitter-az-blob</module>
- <module>tika-emitter-jdbc</module>
- </modules>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-az-blob/pom.xml b/tika-pipes/tika-emitters/tika-emitter-az-blob/pom.xml
deleted file mode 100644
index 5b3e1eb05..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-az-blob/pom.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-az-blob</artifactId>
- <name>Apache Tika Azure blob</name>
-
-
- <dependencies>
- <!-- should serialization be provided or bundled? -->
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-serialization</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.azure</groupId>
- <artifactId>azure-storage-blob</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.emitter.azblob</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/main/java/org/apache/tika/pipes/emitter/azblob/AZBlobEmitter.java b/tika-pipes/tika-emitters/tika-emitter-az-blob/src/main/java/org/apache/tika/pipes/emitter/azblob/AZBlobEmitter.java
deleted file mode 100644
index a4e81f2cc..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/main/java/org/apache/tika/pipes/emitter/azblob/AZBlobEmitter.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.azblob;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Path;
-import java.util.List;
-import java.util.Map;
-
-import com.azure.core.credential.AzureSasCredential;
-import com.azure.storage.blob.BlobClient;
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.BlobServiceClient;
-import com.azure.storage.blob.BlobServiceClientBuilder;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream;
-import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.StreamEmitter;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-import org.apache.tika.serialization.JsonMetadataList;
-import org.apache.tika.utils.StringUtils;
-
-
-/**
- * Emit files to Azure blob storage. Must set endpoint, sasToken and container via config.
- */
-
-public class AZBlobEmitter extends AbstractEmitter implements Initializable, StreamEmitter {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(AZBlobEmitter.class);
- private String fileExtension = "json";
-
- private String prefix = "";
-
- private String sasToken;
- private String container;
- private String endpoint;
- private BlobServiceClient blobServiceClient;
- private BlobContainerClient blobContainerClient;
- private boolean overwriteExisting = false;
-
- /**
- * Requires the src-bucket/path/to/my/file.txt in the {@link TikaCoreProperties#SOURCE_PATH}.
- *
- * @param emitKey
- * @param metadataList
- * @param parseContext
- * @throws IOException
- * @throws TikaEmitterException
- */
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext) throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.isEmpty()) {
- throw new TikaEmitterException("metadata list must not be null or of size 0");
- }
- //TODO: estimate size of metadata list. Above a certain size,
- //create a temp file?
- UnsynchronizedByteArrayOutputStream bos = UnsynchronizedByteArrayOutputStream
- .builder()
- .get();
- try (Writer writer = new OutputStreamWriter(bos, StandardCharsets.UTF_8)) {
- JsonMetadataList.toJson(metadataList, writer);
- } catch (IOException e) {
- throw new TikaEmitterException("can't jsonify", e);
- }
- Metadata metadata = new Metadata();
- emit(emitKey, TikaInputStream.get(bos.toByteArray(), metadata), metadata, parseContext);
-
- }
-
- /**
- * @param path object path; prefix will be prepended
- * @param is inputStream to copy, if a TikaInputStream contains an underlying file,
- * the client will upload the file; if a content-length is included in the
- * metadata, the client will upload the stream with the content-length;
- * otherwise, the client will copy the stream to a byte array and then
- * upload.
- * @param userMetadata this will be written to the az blob's properties.metadata
- * @param parseContext
- * @throws IOException if there is a Runtime client exception
- * @throws TikaEmitterException if there is a Runtime client exception
- */
- @Override
- public void emit(String path, InputStream is, Metadata userMetadata, ParseContext parseContext) throws IOException, TikaEmitterException {
- String lengthString = userMetadata.get(Metadata.CONTENT_LENGTH);
- long length = -1;
- if (lengthString != null) {
- try {
- length = Long.parseLong(lengthString);
- } catch (NumberFormatException e) {
- LOGGER.warn("Bad content-length: " + lengthString);
- }
- }
- if (is instanceof TikaInputStream && ((TikaInputStream) is).hasFile()) {
- write(path, userMetadata, ((TikaInputStream) is).getPath());
- } else if (length > -1) {
- LOGGER.debug("relying on the content-length set in the metadata object: {}", length);
- write(path, userMetadata, is, length);
- } else {
- try (UnsynchronizedByteArrayOutputStream bos = UnsynchronizedByteArrayOutputStream
- .builder()
- .get()) {
- IOUtils.copy(is, bos);
- write(path, userMetadata, bos.toByteArray());
- }
- }
- }
-
- private void write(String path, Metadata userMetadata, InputStream is, long length) {
- String actualPath = getActualPath(path);
- LOGGER.debug("about to emit to target container: ({}) path:({})", container, actualPath);
- BlobClient blobClient = blobContainerClient.getBlobClient(actualPath);
- updateMetadata(blobClient, userMetadata);
- blobClient.upload(is, length, overwriteExisting);
- }
-
- private void write(String path, Metadata userMetadata, Path file) {
- String actualPath = getActualPath(path);
- LOGGER.debug("about to emit to target container: ({}) path:({})", container, actualPath);
- BlobClient blobClient = blobContainerClient.getBlobClient(actualPath);
- updateMetadata(blobClient, userMetadata);
-
- blobClient.uploadFromFile(file
- .toAbsolutePath()
- .toString(), overwriteExisting);
- }
-
- private void write(String path, Metadata userMetadata, byte[] bytes) throws IOException {
- String actualPath = getActualPath(path);
- LOGGER.debug("about to emit to target container: ({}) path:({})", container, actualPath);
- BlobClient blobClient = blobContainerClient.getBlobClient(actualPath);
- updateMetadata(blobClient, userMetadata);
- blobClient.upload(UnsynchronizedByteArrayInputStream.builder().setByteArray(bytes).get(), bytes.length, overwriteExisting);
- }
-
- private void updateMetadata(BlobClient blobClient, Metadata userMetadata) {
- for (String n : userMetadata.names()) {
- if (n.equals(Metadata.CONTENT_LENGTH)) {
- continue;
- }
- String[] vals = userMetadata.getValues(n);
- if (vals.length > 1) {
- LOGGER.warn("Can only write the first value for key {}. I see {} values.", n, vals.length);
- }
- blobClient
- .getProperties()
- .getMetadata()
- .put(n, vals[0]);
- }
-
- }
-
- private String getActualPath(final String path) {
- String ret;
- if (!StringUtils.isBlank(prefix)) {
- ret = prefix + "/" + path;
- } else {
- ret = path;
- }
-
- if (!StringUtils.isBlank(fileExtension)) {
- ret += "." + fileExtension;
- }
- return ret;
- }
-
- @Field
- public void setSasToken(String sasToken) {
- this.sasToken = sasToken;
- }
-
- @Field
- public void setEndpoint(String endpoint) {
- this.endpoint = endpoint;
- }
-
- @Field
- public void setContainer(String container) {
- this.container = container;
- }
-
- @Field
- public void setOverwriteExisting(boolean overwriteExisting) {
- this.overwriteExisting = overwriteExisting;
- }
-
- @Field
- public void setPrefix(String prefix) {
- //strip final "/" if it exists
- if (prefix.endsWith("/")) {
- this.prefix = prefix.substring(0, prefix.length() - 1);
- } else {
- this.prefix = prefix;
- }
- }
-
- /**
- * If you want to customize the output file's file extension.
- * Do not include the "."
- *
- * @param fileExtension
- */
- @Field
- public void setFileExtension(String fileExtension) {
- this.fileExtension = fileExtension;
- }
-
-
- /**
- * This initializes the az blob container client
- *
- * @param params params to use for initialization
- * @throws TikaConfigException
- */
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- //TODO -- allow authentication via other methods
- blobServiceClient = new BlobServiceClientBuilder()
- .endpoint(endpoint)
- .credential(new AzureSasCredential(sasToken))
- .buildClient();
- blobContainerClient = blobServiceClient.getBlobContainerClient(container);
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
- mustNotBeEmpty("sasToken", this.sasToken);
- mustNotBeEmpty("endpoint", this.endpoint);
- mustNotBeEmpty("container", this.container);
- }
-
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/java/org/apache/tika/pipes/emitter/azblob/TestAZBlobEmitter.java b/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/java/org/apache/tika/pipes/emitter/azblob/TestAZBlobEmitter.java
deleted file mode 100644
index bed2367a0..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/java/org/apache/tika/pipes/emitter/azblob/TestAZBlobEmitter.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.azblob;
-
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.Emitter;
-import org.apache.tika.pipes.core.emitter.EmitterManager;
-
-@Disabled("turn into an actual test")
-public class TestAZBlobEmitter {
-
- @Test
- public void testBasic() throws Exception {
- EmitterManager emitterManager = EmitterManager.load(getConfig("tika-config-az-blob.xml"));
- Emitter emitter = emitterManager.getEmitter("az-blob");
- List<Metadata> metadataList = new ArrayList<>();
- Metadata m = new Metadata();
- m.set("k1", "v1");
- m.add("k1", "v2");
- m.set("k2", "v3");
- m.add("k2", "v4");
- metadataList.add(m);
- emitter.emit("something-or-other/test-out", metadataList, new ParseContext());
- }
-
- private Path getConfig(String configFile) throws URISyntaxException {
- return Paths.get(this
- .getClass()
- .getResource("/config/" + configFile)
- .toURI());
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/resources/config/tika-config-az-blob.xml b/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/resources/config/tika-config-az-blob.xml
deleted file mode 100644
index c1ba42b07..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/resources/config/tika-config-az-blob.xml
+++ /dev/null
@@ -1,28 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.azblob.AZBlobEmitter">
- <name>az-blob</name>
- <!-- these have to be non-null -->
- <endpoint></endpoint>
- <container></container>
- <sasToken></sasToken>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-gcs/pom.xml b/tika-pipes/tika-emitters/tika-emitter-gcs/pom.xml
deleted file mode 100644
index afdba7ace..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-gcs/pom.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-gcs</artifactId>
- <name>Apache Tika GCS emitter</name>
-
-
- <dependencies>
- <!-- should serialization be provided or bundled? -->
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-serialization</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.cloud</groupId>
- <artifactId>google-cloud-storage</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.emitter.gcs</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java b/tika-pipes/tika-emitters/tika-emitter-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java
deleted file mode 100644
index fd0f0ce7c..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitter.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.gcs;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.List;
-import java.util.Map;
-
-import com.google.cloud.storage.BlobId;
-import com.google.cloud.storage.BlobInfo;
-import com.google.cloud.storage.Storage;
-import com.google.cloud.storage.StorageOptions;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.StreamEmitter;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-import org.apache.tika.serialization.JsonMetadataList;
-import org.apache.tika.utils.StringUtils;
-
-
-public class GCSEmitter extends AbstractEmitter implements Initializable, StreamEmitter {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GCSEmitter.class);
- private String projectId;
- private String bucket;
- private String fileExtension = "json";
- private String prefix = null;
- private Storage storage;
-
- /**
- * Requires the src-bucket/path/to/my/file.txt in the {@link TikaCoreProperties#SOURCE_PATH}.
- *
- * @param metadataList
- * @throws IOException
- * @throws TikaException
- */
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext) throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.size() == 0) {
- throw new TikaEmitterException("metadata list must not be null or of size 0");
- }
- try (UnsynchronizedByteArrayOutputStream bos = UnsynchronizedByteArrayOutputStream
- .builder()
- .get()) {
- try (Writer writer = new OutputStreamWriter(bos, StandardCharsets.UTF_8)) {
- JsonMetadataList.toJson(metadataList, writer);
- } catch (IOException e) {
- throw new TikaEmitterException("can't jsonify", e);
- }
-
- write(emitKey, new Metadata(), bos.toByteArray());
- }
-
- }
-
- /**
- * @param path -- object path, not including the bucket
- * @param is inputStream to copy
- * @param userMetadata this will be written to the s3 ObjectMetadata's userMetadata
- * @throws TikaEmitterException or IOexception if there is a Runtime s3 client exception
- */
- @Override
- public void emit(String path, InputStream is, Metadata userMetadata, ParseContext parseContext) throws IOException, TikaEmitterException {
-
- if (is instanceof TikaInputStream && ((TikaInputStream) is).hasFile()) {
- write(path, userMetadata, Files.readAllBytes(((TikaInputStream) is).getPath()));
- } else {
- try (UnsynchronizedByteArrayOutputStream bos = UnsynchronizedByteArrayOutputStream
- .builder()
- .get()) {
- IOUtils.copy(is, bos);
- write(path, userMetadata, bos.toByteArray());
- }
- }
- }
-
- private void write(String path, Metadata userMetadata, byte[] bytes) {
- if (!StringUtils.isBlank(prefix)) {
- path = prefix + "/" + path;
- }
-
- if (!StringUtils.isBlank(fileExtension)) {
- path += "." + fileExtension;
- }
-
- LOGGER.debug("about to emit to target bucket: ({}) path:({})", bucket, path);
- BlobId blobId = BlobId.of(bucket, path);
- BlobInfo blobInfo = BlobInfo
- .newBuilder(blobId)
- .build();
-
- for (String n : userMetadata.names()) {
- String[] vals = userMetadata.getValues(n);
- if (vals.length > 1) {
- LOGGER.warn("Can only write the first value for key {}. I see {} values.", n, vals.length);
- }
- blobInfo
- .getMetadata()
- .put(n, vals[0]);
- }
- storage.create(blobInfo, bytes);
- }
-
-
- @Field
- public void setProjectId(String projectId) {
- this.projectId = projectId;
- }
-
- @Field
- public void setBucket(String bucket) {
- this.bucket = bucket;
- }
-
- @Field
- public void setPrefix(String prefix) {
- //strip final "/" if it exists
- if (prefix.endsWith("/")) {
- this.prefix = prefix.substring(0, prefix.length() - 1);
- } else {
- this.prefix = prefix;
- }
- }
-
- /**
- * If you want to customize the output file's file extension.
- * Do not include the "."
- *
- * @param fileExtension
- */
- @Field
- public void setFileExtension(String fileExtension) {
- this.fileExtension = fileExtension;
- }
-
-
- /**
- * This initializes the gcs client.
- *
- * @param params params to use for initialization
- * @throws TikaConfigException
- */
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- //params have already been set...ignore them
- //TODO -- add other params to the builder as needed
- storage = StorageOptions
- .newBuilder()
- .setProjectId(projectId)
- .build()
- .getService();
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
- mustNotBeEmpty("bucket", this.bucket);
- mustNotBeEmpty("projectId", this.projectId);
- }
-
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/java/org/apache/tika/pipes/emitter/gcs/TestGCSEmitter.java b/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/java/org/apache/tika/pipes/emitter/gcs/TestGCSEmitter.java
deleted file mode 100644
index 23c42aa0f..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/java/org/apache/tika/pipes/emitter/gcs/TestGCSEmitter.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.gcs;
-
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.Emitter;
-import org.apache.tika.pipes.core.emitter.EmitterManager;
-
-@Disabled("turn into an actual test")
-public class TestGCSEmitter {
-
- @Test
- public void testBasic() throws Exception {
- EmitterManager emitterManager = EmitterManager.load(getConfig("tika-config-gcs.xml"));
- Emitter emitter = emitterManager.getEmitter("gcs");
- List<Metadata> metadataList = new ArrayList<>();
- Metadata m = new Metadata();
- m.set("k1", "v1");
- m.add("k1", "v2");
- m.set("k2", "v3");
- metadataList.add(m);
- emitter.emit("something-or-other/test-out", metadataList, new ParseContext());
- }
-
- private Path getConfig(String configFile) throws URISyntaxException {
- return Paths.get(this
- .getClass()
- .getResource("/config/" + configFile)
- .toURI());
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/resources/config/tika-config-gcs.xml b/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/resources/config/tika-config-gcs.xml
deleted file mode 100644
index 2ea06761e..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/resources/config/tika-config-gcs.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.gcs.GCSEmitter">
- <name>gcs</name>
- <projectId>My First Project</projectId>
- <bucket>tika-tallison-test-bucket</bucket>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/pom.xml b/tika-pipes/tika-emitters/tika-emitter-jdbc/pom.xml
deleted file mode 100644
index 6698b624d..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/pom.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-jdbc</artifactId>
- <name>Apache Tika jdbc emitter</name>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.emitter.jdbc</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/main/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitter.java b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/main/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitter.java
deleted file mode 100644
index ced0a92ff..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/main/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitter.java
+++ /dev/null
@@ -1,701 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.jdbc;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.sql.Timestamp;
-import java.sql.Types;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.EmitData;
-import org.apache.tika.pipes.core.emitter.EmitKey;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-import org.apache.tika.utils.StringUtils;
-
-/**
- * This is only an initial, basic implementation of an emitter for JDBC.
- * <p>
- * It is currently NOT thread safe because of the shared prepared statement,
- * and depending on the jdbc implementation because of the shared connection.
- * <p>
- * As of the 2.5.0 release, this is ALPHA version. There may be breaking changes
- * in the future.
- */
-public class JDBCEmitter extends AbstractEmitter implements Initializable, Closeable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(JDBCEmitter.class);
-
- public enum AttachmentStrategy {
- FIRST_ONLY, ALL
- //anything else?
- }
-
- public enum MultivaluedFieldStrategy {
- FIRST_ONLY, CONCATENATE
- //anything else?
- }
-
- //some file formats do not have time zones...
- //try both
- private static final String[] TIKA_DATE_PATTERNS =
- new String[] {"yyyy-MM-dd'T'HH:mm:ss'Z'", "yyyy-MM-dd'T'HH:mm:ss"};
- //the "write" lock is used for creating the table
- private static ReadWriteLock READ_WRITE_LOCK = new ReentrantReadWriteLock();
- //this keeps track of which table + connection string have been created
- //so that only one table is created per table + connection string.
- //This is necessary for testing and if someone specifies multiple
- //different jdbc emitters.
- private static Set<String> TABLES_CREATED = new HashSet<>();
- private String connectionString;
-
- private Optional<String> postConnectionString = Optional.empty();
- private String insert;
- private String createTable;
- private String alterTable;
-
- private int maxRetries = 0;
-
- //used only for specification of column name/string definition of
- //keys
- private Map<String, String> keys;
-
- private List<ColumnDefinition> columns;
- private Connection connection;
- private PreparedStatement insertStatement;
- private AttachmentStrategy attachmentStrategy = AttachmentStrategy.FIRST_ONLY;
-
- private MultivaluedFieldStrategy multivaluedFieldStrategy =
- MultivaluedFieldStrategy.CONCATENATE;
-
- private String multivaluedFieldDelimiter = ", ";
-
- //emitters are run in a single thread. If we ever start running them
- //multithreaded, this will be a big problem.
- private final DateFormat[] dateFormats;
-
- private int maxStringLength = 64000;
-
- //this is set during the initialize phase
- private StringNormalizer stringNormalizer;
-
- public JDBCEmitter() {
- dateFormats = new DateFormat[TIKA_DATE_PATTERNS.length];
- int i = 0;
- for (String p : TIKA_DATE_PATTERNS) {
- dateFormats[i++] = new SimpleDateFormat(p, Locale.US);
- }
- }
-
- /**
- * This is called immediately after the table is created.
- * The purpose of this is to allow for adding a complex primary key or
- * other constraint on the table after it is created.
- *
- * @param alterTable
- */
- public void setAlterTable(String alterTable) {
- this.alterTable = alterTable;
- }
-
- @Field
- public void setCreateTable(String createTable) {
- this.createTable = createTable;
- }
-
- @Field
- public void setInsert(String insert) {
- this.insert = insert;
- }
-
- @Field
- public void setConnection(String connection) {
- this.connectionString = connection;
- }
-
- /**
- * Set the maximum string length in characters (not bytes).
- * This is applies only to fields with name "string"
- * not to "varchar".
- *
- * @param maxStringLength
- */
- @Field
- public void setMaxStringLength(int maxStringLength) {
- this.maxStringLength = maxStringLength;
- }
-
- public void setMaxRetries(int maxRetries) {
- this.maxRetries = maxRetries;
- }
-
- /**
- * This sql will be called immediately after the connection is made. This was
- * initially added for setting pragmas on sqlite3, but may be used for other
- * connection configuration in other dbs. Note: This is called before the table is
- * created if it needs to be created.
- *
- * @param postConnection
- */
- @Field
- public void setPostConnection(String postConnection) {
- this.postConnectionString = Optional.of(postConnection);
- }
-
- /**
- * This applies to fields of type 'string' or 'varchar'. If there's
- * a multivalued field in a metadata object, do you want the first value only
- * or should we concatenate these with the
- * {@link JDBCEmitter#setMultivaluedFieldDelimiter(String)}.
- * <p>
- * The default values as of 2.6.1 are {@link MultivaluedFieldStrategy#CONCATENATE}
- * and the default delimiter is ", "
- *
- * @param strategy
- * @throws TikaConfigException
- */
- @Field
- public void setMultivaluedFieldStrategy(String strategy) throws TikaConfigException {
- String lc = strategy.toLowerCase(Locale.US);
- if (lc.equals("first_only")) {
- setMultivaluedFieldStrategy(MultivaluedFieldStrategy.FIRST_ONLY);
- } else if (lc.equals("concatenate")) {
- setMultivaluedFieldStrategy(MultivaluedFieldStrategy.CONCATENATE);
- } else {
- throw new TikaConfigException("I'm sorry, I only recogize 'first_only' and " +
- "'concatenate'. I don't mind '" + strategy + "'");
- }
- }
-
- public void setMultivaluedFieldStrategy(MultivaluedFieldStrategy multivaluedFieldStrategy) {
- this.multivaluedFieldStrategy = multivaluedFieldStrategy;
- }
-
- /**
- * See {@link JDBCEmitter#setMultivaluedFieldDelimiter(String)}
- *
- * @param delimiter
- */
- @Field
- public void setMultivaluedFieldDelimiter(String delimiter) {
- this.multivaluedFieldDelimiter = delimiter;
- }
-
- /**
- * The implementation of keys should be a LinkedHashMap because
- * order matters!
- * <p>
- * Key is the name of the metadata field, value is the type of column:
- * boolean, string, int, long
- *
- * @param keys
- */
- @Field
- public void setKeys(Map<String, String> keys) {
- this.keys = keys;
- }
-
- public void setAttachmentStrategy(AttachmentStrategy attachmentStrategy) {
- this.attachmentStrategy = attachmentStrategy;
- }
-
- @Field
- public void setAttachmentStrategy(String attachmentStrategy) {
- if ("all".equalsIgnoreCase(attachmentStrategy)) {
- setAttachmentStrategy(AttachmentStrategy.ALL);
- } else if ("first_only".equalsIgnoreCase(attachmentStrategy)) {
- setAttachmentStrategy(AttachmentStrategy.FIRST_ONLY);
- } else {
- throw new IllegalArgumentException("attachmentStrategy must be 'all' or 'first_only'");
- }
- }
-
- /**
- * This executes the emit with each call. For more efficient
- * batch execution use {@link #emit(List)}.
- *
- * @param emitKey emit key
- * @param metadataList list of metadata per file
- * @throws IOException
- * @throws TikaEmitterException
- */
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext)
- throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.size() < 1) {
- return;
- }
- List<EmitData> emitDataList = new ArrayList<>();
- emitDataList.add(new EmitData(new EmitKey("", emitKey), metadataList));
- emit(emitDataList);
- }
-
- @Override
- public void emit(List<? extends EmitData> emitData) throws IOException, TikaEmitterException {
- int tries = 0;
- Exception ex = null;
- while (tries++ <= maxRetries) {
- try {
- emitNow(emitData);
- return;
- } catch (SQLException e) {
- try {
- reconnect();
- } catch (SQLException exc) {
- throw new TikaEmitterException("couldn't reconnect!", exc);
- }
- ex = e;
- }
- }
- throw new TikaEmitterException("Couldn't emit " + emitData.size() + " records.", ex);
- }
-
- private void emitNow(List<? extends EmitData> emitData) throws SQLException {
- if (attachmentStrategy == AttachmentStrategy.FIRST_ONLY) {
- for (EmitData d : emitData) {
- insertFirstOnly(d.getEmitKey().getEmitKey(), d.getMetadataList());
- insertStatement.addBatch();
- }
- } else {
- for (EmitData d : emitData) {
- insertAll(d.getEmitKey().getEmitKey(), d.getMetadataList());
- }
- }
- if (LOGGER.isDebugEnabled()) {
- long start = System.currentTimeMillis();
- insertStatement.executeBatch();
- connection.commit();
- LOGGER.debug("took {}ms to insert {} rows ", System.currentTimeMillis() - start,
- emitData.size());
- } else {
- insertStatement.executeBatch();
- connection.commit();
- }
-
- }
-
- private void insertAll(String emitKey, List<Metadata> metadataList) throws SQLException {
-
- for (int i = 0; i < metadataList.size(); i++) {
- insertStatement.clearParameters();
- int col = 0;
- insertStatement.setString(++col, emitKey);
- insertStatement.setInt(++col, i);
- for (ColumnDefinition columnDefinition : columns) {
- updateValue(emitKey, insertStatement, ++col, columnDefinition, i, metadataList);
- }
- insertStatement.addBatch();
- }
- }
-
- private void insertFirstOnly(String emitKey, List<Metadata> metadataList) throws SQLException {
- insertStatement.clearParameters();
- int i = 0;
- insertStatement.setString(++i, emitKey);
- for (ColumnDefinition columnDefinition : columns) {
- updateValue(emitKey, insertStatement, ++i, columnDefinition, 0, metadataList);
- }
- }
-
- private void reconnect() throws SQLException {
- SQLException ex = null;
- for (int i = 0; i < 3; i++) {
- try {
- tryClose();
- createConnection();
- insertStatement = connection.prepareStatement(insert);
- return;
- } catch (SQLException e) {
- LOGGER.warn("couldn't reconnect to db", e);
- ex = e;
- }
- }
- throw ex;
- }
-
- private void tryClose() {
- if (insertStatement != null) {
- try {
- insertStatement.close();
- } catch (SQLException e) {
- LOGGER.warn("exception closing insert", e);
- }
- }
-
- if (connection != null) {
- try {
- connection.commit();
- connection.close();
- } catch (SQLException e) {
- LOGGER.warn("exception closing connection", e);
- }
- }
- }
-
- private void createConnection() throws SQLException {
- connection = DriverManager.getConnection(connectionString);
- connection.setAutoCommit(false);
- if (postConnectionString.isPresent()) {
- try (Statement st = connection.createStatement()) {
- st.execute(postConnectionString.get());
- }
- }
- }
-
- private void updateValue(String emitKey, PreparedStatement insertStatement, int i,
- ColumnDefinition columnDefinition, int metadataListIndex,
- List<Metadata> metadataList)
- throws SQLException {
- Metadata metadata = metadataList.get(metadataListIndex);
- String val = getVal(metadata, columnDefinition);
- switch (columnDefinition.getType()) {
- case Types.VARCHAR:
- updateVarchar(emitKey, columnDefinition, insertStatement, i, val);
- break;
- case Types.BOOLEAN:
- updateBoolean(insertStatement, i, val);
- break;
- case Types.INTEGER:
- updateInteger(insertStatement, i, val);
- break;
- case Types.BIGINT:
- updateLong(insertStatement, i, val);
- break;
- case Types.FLOAT:
- updateFloat(insertStatement, i, val);
- break;
- case Types.DOUBLE:
- updateDouble(insertStatement, i, val);
- break;
- case Types.TIMESTAMP:
- updateTimestamp(insertStatement, i, val, dateFormats);
- break;
- default:
- throw new IllegalArgumentException(
- "Can only process:" + getHandledTypes() +
- " types so far. " +
- "Please open a ticket to request: " +
- columnDefinition.getType() + " for " +
- columnDefinition.getColumnName());
- }
- }
-
- private String getVal(Metadata metadata, ColumnDefinition columnDefinition) {
- if (columnDefinition.getType() != Types.VARCHAR) {
- return metadata.get(columnDefinition.getColumnName());
- }
- if (multivaluedFieldStrategy == MultivaluedFieldStrategy.FIRST_ONLY) {
- return metadata.get(columnDefinition.getColumnName());
- }
- String[] vals = metadata.getValues(columnDefinition.getColumnName());
- if (vals.length == 0) {
- return null;
- } else if (vals.length == 1) {
- return vals[0];
- }
-
- int i = 0;
- StringBuilder sb = new StringBuilder();
- for (String val : metadata.getValues(columnDefinition.getColumnName())) {
- if (StringUtils.isBlank(val)) {
- continue;
- }
- if (i > 0) {
- sb.append(multivaluedFieldDelimiter);
- }
- sb.append(val);
- i++;
- }
- return sb.toString();
- }
-
- private void updateDouble(PreparedStatement insertStatement, int i, String val)
- throws SQLException {
- if (StringUtils.isBlank(val)) {
- insertStatement.setNull(i, Types.DOUBLE);
- return;
- }
- Double d = Double.parseDouble(val);
- insertStatement.setDouble(i, d);
- }
-
- private void updateVarchar(String emitKey, ColumnDefinition columnDefinition, PreparedStatement insertStatement,
- int i,
- String val) throws SQLException {
- if (val == null) {
- insertStatement.setNull(i, Types.VARCHAR);
- return;
- }
- String normalized = stringNormalizer.normalize(emitKey,
- columnDefinition.getColumnName(), val, columnDefinition.getPrecision());
- insertStatement.setString(i, normalized);
- }
-
- private void updateTimestamp(PreparedStatement insertStatement, int i, String val,
- DateFormat[] dateFormats) throws SQLException {
- if (StringUtils.isBlank(val)) {
- insertStatement.setNull(i, Types.TIMESTAMP);
- return;
- }
-
- for (DateFormat df : dateFormats) {
- try {
- Date d = df.parse(val);
- insertStatement.setTimestamp(i, new Timestamp(d.getTime()));
- return;
- } catch (ParseException e) {
- //ignore
- }
- }
- LOGGER.warn("Couldn't parse {}" + val);
- insertStatement.setNull(i, Types.TIMESTAMP);
- }
-
- private void updateFloat(PreparedStatement insertStatement, int i, String val)
- throws SQLException {
- if (StringUtils.isBlank(val)) {
- insertStatement.setNull(i, Types.FLOAT);
- } else {
- insertStatement.setFloat(i, Float.parseFloat(val));
- }
- }
-
- private void updateLong(PreparedStatement insertStatement, int i, String val)
- throws SQLException {
- if (StringUtils.isBlank(val)) {
- insertStatement.setNull(i, Types.BIGINT);
- } else {
- insertStatement.setLong(i, Long.parseLong(val));
- }
- }
-
- private void updateInteger(PreparedStatement insertStatement, int i, String val)
- throws SQLException {
- if (StringUtils.isBlank(val)) {
- insertStatement.setNull(i, Types.INTEGER);
- } else {
- insertStatement.setInt(i, Integer.parseInt(val));
- }
- }
-
- private void updateBoolean(PreparedStatement insertStatement, int i, String val)
- throws SQLException {
- if (StringUtils.isBlank(val)) {
- insertStatement.setNull(i, Types.BOOLEAN);
- } else {
- insertStatement.setBoolean(i, Boolean.parseBoolean(val));
- }
- }
-
-
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- parseColTypes();
- setStringNormalizer();
- try {
- createConnection();
- } catch (SQLException e) {
- throw new TikaConfigException("couldn't open connection: " + connectionString, e);
- }
- if (!StringUtils.isBlank(createTable)) {
- //synchronize table creation
- READ_WRITE_LOCK.writeLock().lock();
- try {
- String tableCreationString = connectionString + " " + createTable;
- if (!TABLES_CREATED.contains(tableCreationString)) {
- try (Statement st = connection.createStatement()) {
- st.execute(createTable);
- if (!StringUtils.isBlank(alterTable)) {
- st.execute(alterTable);
- }
- TABLES_CREATED.add(tableCreationString);
- } catch (SQLException e) {
- throw new TikaConfigException("can't create table", e);
- }
- }
- } finally {
- READ_WRITE_LOCK.writeLock().unlock();
- }
- }
- try {
- insertStatement = connection.prepareStatement(insert);
- } catch (SQLException e) {
- throw new TikaConfigException("can't create insert statement", e);
- }
- }
-
- private void setStringNormalizer() {
- if (connectionString.startsWith("jdbc:postgres")) {
- stringNormalizer = new JDBCEmitter.PostgresNormalizer();
- } else {
- stringNormalizer = new JDBCEmitter.StringNormalizer();
- }
- }
-
- private void parseColTypes() {
- columns = new ArrayList<>();
- for (Map.Entry<String, String> e : keys.entrySet()) {
- columns.add(ColumnDefinition.parse(e.getKey(), e.getValue(), maxStringLength));
- }
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler)
- throws TikaConfigException {
- //require
- }
-
- /**
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- try {
- insertStatement.close();
- } catch (SQLException e) {
- LOGGER.warn("problem closing insert", e);
- }
- try {
- connection.close();
- } catch (SQLException e) {
- throw new IOException(e);
- }
- }
-
- private static String getHandledTypes() {
- return "'string', 'varchar', " +
- "'boolean', 'int', 'long', 'float', 'double' and 'timestamp'";
- }
-
- private static class StringNormalizer {
-
- String normalize(String emitKey, String columnName, String s, int maxLength) {
- if (maxLength < 0 || s.length() <= maxLength) {
- return s;
- }
- LOGGER.warn("truncating {}->'{}' from {} chars to {} chars",
- emitKey, columnName, s.length(), maxLength);
-
- return s.substring(0, maxLength);
- }
- }
-
- private static class PostgresNormalizer extends StringNormalizer {
-
- @Override
- String normalize(String emitKey, String columnName, String s, int maxLength) {
- s = s.replaceAll("\u0000", " ");
- return super.normalize(emitKey, columnName, s, maxLength);
- }
- }
-
- private static class ColumnDefinition {
- private static final Matcher VARCHAR_MATCHER =
- Pattern.compile("varchar\\((\\d+)\\)").matcher("");
-
- private final String columnName;
-
- private final int type;
- //this is only used (so far) for varchar. It is currently
- //ignored for other data types
- private final int precision;
-
- private static ColumnDefinition parse(String name, String type, int maxStringLength) {
- String lcType = type.toLowerCase(Locale.US);
- if (VARCHAR_MATCHER.reset(lcType).find()) {
- return new ColumnDefinition(name,
- Types.VARCHAR, Integer.parseInt(VARCHAR_MATCHER.group(1)));
- }
-
- switch (lcType) {
-
- case "string":
- return new ColumnDefinition(name, Types.VARCHAR, maxStringLength);
- case "bool":
- case "boolean":
- return new ColumnDefinition(name, Types.BOOLEAN, -1);
- case "int":
- case "integer":
- return new ColumnDefinition(name, Types.INTEGER, -1);
- case "bigint":
- case "long":
- return new ColumnDefinition(name, Types.BIGINT, -1);
- case "float":
- return new ColumnDefinition(name, Types.FLOAT, -1);
- case "double":
- return new ColumnDefinition(name, Types.DOUBLE, -1);
- case "timestamp":
- return new ColumnDefinition(name, Types.TIMESTAMP, -1);
-
- default:
- throw new IllegalArgumentException(
- "Can only process: " + getHandledTypes() +
- " types so far. Please open a ticket to request " +
- type + " for column: " + name);
- }
- }
-
- private ColumnDefinition(String columnName, int type, int precision) {
- this.columnName = columnName;
- this.type = type;
- this.precision = precision;
- }
-
- public String getColumnName() {
- return columnName;
- }
-
- public int getType() {
- return type;
- }
-
- public int getPrecision() {
- return precision;
- }
- }
-
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java
deleted file mode 100644
index 448484197..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.jdbc;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.Statement;
-import java.sql.Timestamp;
-import java.time.ZoneId;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.Emitter;
-import org.apache.tika.pipes.core.emitter.EmitterManager;
-
-public class JDBCEmitterTest {
-
- @Test
- public void testBasic(@TempDir Path tmpDir) throws Exception {
- Files.createDirectories(tmpDir.resolve("db"));
- Path dbDir = tmpDir.resolve("db/h2");
- Path config = tmpDir.resolve("tika-config.xml");
- String connectionString = "jdbc:h2:file:" + dbDir.toAbsolutePath();
-
- writeConfig("/configs/tika-config-jdbc-emitter.xml",
- connectionString, config);
-
- EmitterManager emitterManager = EmitterManager.load(config);
- Emitter emitter = emitterManager.getEmitter();
- List<String[]> data = new ArrayList<>();
- data.add(new String[]{"k1", "true", "k2", "some string1", "k3", "4", "k4", "100"});
- data.add(new String[]{"k1", "false", "k2", "some string2", "k3", "5", "k4", "101"});
- data.add(new String[]{"k1", "true", "k2", "some string3", "k3", "6", "k4", "102"});
- //test dates with and without timezones
- data.add(new String[]{"k1", "false", "k2", "some string4", "k3", "7", "k4", "103", "k5",
- "100002", "k6", "2022-11-04T17:10:15Z"});
-
- data.add(new String[]{"k1", "true", "k2", "some string5", "k3", "8", "k4", "104", "k5",
- "100002", "k6", "2022-11-04T17:10:15"});
- int id = 0;
- for (String[] d : data) {
- emitter.emit("id" + id++, Collections.singletonList(m(d)), new ParseContext());
- }
-
- try (Connection connection = DriverManager.getConnection(connectionString)) {
- try (Statement st = connection.createStatement()) {
- try (ResultSet rs = st.executeQuery("select * from test")) {
- int rows = 0;
- while (rs.next()) {
- assertEquals("id" + rows, rs.getString(1));
- assertEquals(rows % 2 == 0, rs.getBoolean(2));
- assertEquals("some string" + (rows + 1), rs.getString(3));
- assertEquals(rows + 4, rs.getInt(4));
- assertEquals(100 + rows, rs.getLong(5));
- if (rows > 2) {
- assertEquals(100002, rs.getLong(6));
- Timestamp timestamp = rs.getTimestamp(7);
- String str = timestamp.toInstant().atZone(ZoneId.of("UTC")).toString();
- //TODO fix this to work in other timezones
- assertTrue(str.startsWith("2022-11"));
- }
- rows++;
- }
- }
- }
- }
- }
-
- @Test
- public void testTableExists(@TempDir Path tmpDir) throws Exception {
- String createTable = "create table test (path varchar(512) primary key," +
- "k1 boolean,k2 varchar(512),k3 integer,k4 long);";
-
- Files.createDirectories(tmpDir.resolve("db"));
- Path dbDir = tmpDir.resolve("db/h2");
- Path config = tmpDir.resolve("tika-config.xml");
- String connectionString = "jdbc:h2:file:" + dbDir.toAbsolutePath();
- writeConfig("/configs/tika-config-jdbc-emitter-existing-table.xml",
- connectionString, config);
-
- try (Connection connection = DriverManager.getConnection(connectionString)) {
- connection.createStatement().execute(createTable);
- }
- EmitterManager emitterManager = EmitterManager.load(config);
- Emitter emitter = emitterManager.getEmitter();
- List<String[]> data = new ArrayList<>();
- data.add(new String[]{"k1", "true", "k2", "some string1", "k3", "4", "k4", "100"});
- data.add(new String[]{"k1", "false", "k2", "some string2", "k3", "5", "k4", "101"});
- data.add(new String[]{"k1", "true", "k2", "some string3", "k3", "6", "k4", "102"});
- int id = 0;
- for (String[] d : data) {
- emitter.emit("id" + id++, Collections.singletonList(m(d)), new ParseContext());
- }
-
- try (Connection connection = DriverManager.getConnection(connectionString)) {
- try (Statement st = connection.createStatement()) {
- try (ResultSet rs = st.executeQuery("select * from test")) {
- int rows = 0;
- while (rs.next()) {
- assertEquals("id" + rows, rs.getString(1));
- assertEquals(rows % 2 == 0, rs.getBoolean(2));
- assertEquals("some string" + (rows + 1), rs.getString(3));
- assertEquals(rows + 4, rs.getInt(4));
- assertEquals(100 + rows, rs.getLong(5));
- rows++;
- }
- }
- }
- }
- }
-
- @Test
- public void testAttachments(@TempDir Path tmpDir) throws Exception {
- Files.createDirectories(tmpDir.resolve("db"));
- Path dbDir = tmpDir.resolve("db/h2");
- Path config = tmpDir.resolve("tika-config.xml");
- String connectionString = "jdbc:h2:file:" + dbDir.toAbsolutePath();
-
- writeConfig("/configs/tika-config-jdbc-emitter-attachments.xml",
- connectionString, config);
-
- EmitterManager emitterManager = EmitterManager.load(config);
- Emitter emitter = emitterManager.getEmitter();
- List<Metadata> data = new ArrayList<>();
- data.add(m("k1", "true", "k2", "some string1", "k3", "4", "k4", "100"));
- data.add(m("k1", "false", "k2", "some string2", "k3", "5", "k4", "101"));
- data.add(m("k1", "true", "k2", "some string3", "k3", "6", "k4", "102"));
- emitter.emit("id0", data, new ParseContext());
-
-
- try (Connection connection = DriverManager.getConnection(connectionString)) {
- try (Statement st = connection.createStatement()) {
- try (ResultSet rs = st.executeQuery("select * from test")) {
- int rows = 0;
- assertEquals("path", rs.getMetaData().getColumnName(1).toLowerCase(Locale.US));
- assertEquals("attachment_num",
- rs.getMetaData().getColumnName(2).toLowerCase(Locale.US));
- while (rs.next()) {
- assertEquals("id0", rs.getString(1));
- assertEquals(rows, rs.getInt(2));
- assertEquals(rows % 2 == 0, rs.getBoolean(3));
- assertEquals("some string" + (rows + 1), rs.getString(4));
- assertEquals(rows + 4, rs.getInt(5));
- assertEquals(100 + rows, rs.getLong(6));
- rows++;
- }
- }
- }
- }
- }
-
- @Test
- public void testMultiValuedFields(@TempDir Path tmpDir) throws Exception {
- Files.createDirectories(tmpDir.resolve("db"));
- Path dbDir = tmpDir.resolve("db/h2");
- Path config = tmpDir.resolve("tika-config.xml");
- String connectionString = "jdbc:h2:file:" + dbDir.toAbsolutePath();
-
- writeConfig("/configs/tika-config-jdbc-emitter-multivalued.xml",
- connectionString, config);
-
- EmitterManager emitterManager = EmitterManager.load(config);
- Emitter emitter = emitterManager.getEmitter();
- List<Metadata> data = new ArrayList<>();
- Metadata m = new Metadata();
- m.add("k1", "first");
- m.add("k1", "second");
- m.add("k1", "third");
- m.add("k1", "fourth");
- data.add(m);
- emitter.emit("id0", data, new ParseContext());
-
- String expected = "first, second, third, fourth";
- int rows = 0;
- try (Connection connection = DriverManager.getConnection(connectionString)) {
- try (Statement st = connection.createStatement()) {
- try (ResultSet rs = st.executeQuery("select * from test")) {
- assertEquals("path", rs.getMetaData().getColumnName(1).toLowerCase(Locale.US));
- while (rs.next()) {
- assertEquals("id0", rs.getString(1));
- assertEquals(expected, rs.getString(2));
- rows++;
- }
- }
- }
- }
- assertEquals(1, rows);
- }
-
- @Test
- public void testVarcharTruncation(@TempDir Path tmpDir) throws Exception {
- Files.createDirectories(tmpDir.resolve("db"));
- Path dbDir = tmpDir.resolve("db/h2");
- Path config = tmpDir.resolve("tika-config.xml");
- String connectionString = "jdbc:h2:file:" + dbDir.toAbsolutePath();
-
- writeConfig("/configs/tika-config-jdbc-emitter-trunc.xml",
- connectionString, config);
-
- EmitterManager emitterManager = EmitterManager.load(config);
- Emitter emitter = emitterManager.getEmitter();
- List<String[]> data = new ArrayList<>();
- data.add(new String[]{"k1", "abcd"});
- data.add(new String[]{"k1", "abcdefghijklmnopqrs"});
- data.add(new String[]{"k1", "abcdefghijk"});
- int id = 0;
- for (String[] d : data) {
- emitter.emit("id" + id++, Collections.singletonList(m(d)), new ParseContext());
- }
-
- int rows = 0;
- try (Connection connection = DriverManager.getConnection(connectionString)) {
- try (Statement st = connection.createStatement()) {
- try (ResultSet rs = st.executeQuery("select * from test")) {
- while (rs.next()) {
- String s = rs.getString(2);
- assertTrue(s.length() < 13);
- assertFalse(s.contains("m"));
- rows++;
- }
- }
- }
- }
- assertEquals(3, rows);
- }
-
- private void writeConfig(String srcConfig, String dbDir, Path config) throws IOException {
- String xml = IOUtils.resourceToString(srcConfig, StandardCharsets.UTF_8);
- xml = xml.replace("CONNECTION_STRING", dbDir);
- Files.write(config, xml.getBytes(StandardCharsets.UTF_8));
- }
-
- private Metadata m(String... strings) {
- Metadata metadata = new Metadata();
- for (int i = 0; i < strings.length; i++) {
- metadata.set(strings[i], strings[++i]);
- }
- return metadata;
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-attachments.xml b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-attachments.xml
deleted file mode 100644
index 4bc2d8e87..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-attachments.xml
+++ /dev/null
@@ -1,53 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.jdbc.JDBCEmitter">
- <name>jdbc</name>
- <connection>CONNECTION_STRING</connection>
- <createTable>create table test
- (path varchar(512) not null,
- attachment_num integer not null,
- k1 boolean,
- k2 varchar(512),
- k3 integer,
- k4 long);
- </createTable>
- <alterTable>alter table test add primary key (path, attachment_num)</alterTable>
- <!-- the jdbc emitter always puts ths emitKey value as the first
- item -->
- <insert>insert into test (path, attachment_num, k1, k2, k3, k4) values (?,?,?,?,?,?);
- </insert>
- <!-- these are the keys in the metadata object.
- The emitKey is added as the first element in the insert statement.
- Then the these values are added in order.
- They must be in the order of the insert statement.
- The emit key is added as
- -->
- <keys>
- <key k="k1" v="boolean"/>
- <key k="k2" v="string"/>
- <key k="k3" v="int"/>
- <key k="k4" v="long"/>
- </keys>
- <attachmentStrategy>all</attachmentStrategy>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-existing-table.xml b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-existing-table.xml
deleted file mode 100644
index 654b279be..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-existing-table.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.jdbc.JDBCEmitter">
- <name>jdbc</name>
- <connection>CONNECTION_STRING</connection>
- <insert>insert into test (path, k1, k2, k3, k4) values (?,?,?,?,?);
- </insert>
- <!-- these are the keys in the metadata object.
- The emitKey is added as the first element in the insert statement.
- Then the these values are added in order.
- They must be in the order of the insert statement.
- The emit key is added as
- -->
- <keys>
- <key k="k1" v="boolean"/>
- <key k="k2" v="string"/>
- <key k="k3" v="int"/>
- <key k="k4" v="long"/>
- </keys>
- <attachmentStrategy>first_only</attachmentStrategy>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-multivalued.xml b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-multivalued.xml
deleted file mode 100644
index eb966b54a..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-multivalued.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.jdbc.JDBCEmitter">
- <name>jdbc</name>
- <connection>CONNECTION_STRING</connection>
- <createTable>create table test
- (path varchar(512) primary key,
- k1 varchar(512));
- </createTable>
- <!-- the jdbc emitter always puts ths emitKey value as the first
- item -->
- <insert>insert into test (path, k1) values (?,?);
- </insert>
- <!-- these are the keys in the metadata object.
- The emitKey is added as the first element in the insert statement.
- Then the these values are added in order.
- They must be in the order of the insert statement.
- -->
- <keys>
- <key k="k1" v="varchar(512)"/>
- </keys>
- <multivaluedFieldStrategy>concatenate</multivaluedFieldStrategy>
- <multivaluedFieldDelimiter>, </multivaluedFieldDelimiter>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-trunc.xml b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-trunc.xml
deleted file mode 100644
index 85eef281c..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter-trunc.xml
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.jdbc.JDBCEmitter">
- <name>jdbc</name>
- <connection>CONNECTION_STRING</connection>
- <createTable>create table test
- (path varchar(512) primary key,
- k1 varchar(12));
- </createTable>
- <!-- the jdbc emitter always puts ths emitKey value as the first
- item -->
- <insert>insert into test (path, k1) values (?,?);
- </insert>
- <!-- these are the keys in the metadata object.
- The emitKey is added as the first element in the insert statement.
- Then the these values are added in order.
- They must be in the order of the insert statement.
- -->
- <keys>
- <key k="k1" v="varchar(12)"/>
- </keys>
- <attachmentStrategy>first_only</attachmentStrategy>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter.xml b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter.xml
deleted file mode 100644
index c1a05bdec..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/resources/configs/tika-config-jdbc-emitter.xml
+++ /dev/null
@@ -1,54 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.jdbc.JDBCEmitter">
- <name>jdbc</name>
- <connection>CONNECTION_STRING</connection>
- <createTable>create table test
- (path varchar(512) primary key,
- k1 boolean,
- k2 varchar(512),
- k3 integer,
- k4 long,
- k5 bigint,
- k6 timestamp);
- </createTable>
- <!-- the jdbc emitter always puts ths emitKey value as the first
- item -->
- <insert>insert into test (path, k1, k2, k3, k4, k5, k6) values (?,?,?,?,?,?,?);
- </insert>
- <!-- these are the keys in the metadata object.
- The emitKey is added as the first element in the insert statement.
- Then the these values are added in order.
- They must be in the order of the insert statement.
- -->
- <keys>
- <key k="k1" v="boolean"/>
- <key k="k2" v="string"/>
- <key k="k3" v="int"/>
- <key k="k4" v="long"/>
- <key k="k5" v="bigint"/>
- <key k="k6" v="timestamp"/>
- </keys>
- <attachmentStrategy>first_only</attachmentStrategy>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-kafka/pom.xml b/tika-pipes/tika-emitters/tika-emitter-kafka/pom.xml
deleted file mode 100644
index dd7b2d2de..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-kafka/pom.xml
+++ /dev/null
@@ -1,118 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-kafka</artifactId>
- <name>Apache Tika Kafka emitter</name>
-
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${kafka.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.emitter.kafka</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-kafka/src/main/java/org/apache/tika/pipes/emitter/kafka/KafkaEmitter.java b/tika-pipes/tika-emitters/tika-emitter-kafka/src/main/java/org/apache/tika/pipes/emitter/kafka/KafkaEmitter.java
deleted file mode 100644
index cc7f4ea67..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-kafka/src/main/java/org/apache/tika/pipes/emitter/kafka/KafkaEmitter.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.kafka;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerConfig;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-
-/**
- * Emits the now-parsed documents into a specified Apache Kafka topic.
- */
-public class KafkaEmitter extends AbstractEmitter implements Initializable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(KafkaEmitter.class);
-
- private static final ObjectMapper OM = new ObjectMapper();
-
- String topic;
- String bootstrapServers;
-
- String acks = "all";
- int lingerMs = 5000;
- int batchSize = 16384;
- int bufferMemory = 32 * 1024 * 1024;
- String compressionType = "none";
- int connectionsMaxIdleMs = 9 * 60 * 1000;
- int deliveryTimeoutMs = 120 * 1000;
- boolean enableIdempotence = false;
- String interceptorClasses;
- int maxBlockMs = 60 * 1000;
- int maxInFlightRequestsPerConnection = 5;
- int maxRequestSize = 1024 * 1024;
- int metadataMaxAgeMs = 5 * 60 * 1000;
- int requestTimeoutMs = 30 * 1000;
- int retries = Integer.MAX_VALUE;
- int retryBackoffMs = 100;
- int transactionTimeoutMs = 60000;
- String transactionalId;
- String clientId;
- String keySerializer;
- String valueSerializer;
-
- private Producer<String, String> producer;
-
- @Field
- public void setBootstrapServers(String bootstrapServers) {
- this.bootstrapServers = bootstrapServers;
- }
-
- @Field
- public void setAcks(String acks) {
- this.acks = acks;
- }
-
- @Field
- public void setLingerMs(int lingerMs) {
- this.lingerMs = lingerMs;
- }
-
- public void setBatchSize(int batchSize) {
- this.batchSize = batchSize;
- }
-
- @Field
- public void setBufferMemory(int bufferMemory) {
- this.bufferMemory = bufferMemory;
- }
-
- @Field
- public void setClientId(String clientId) {
- this.clientId = clientId;
- }
-
- @Field
- public void setCompressionType(String compressionType) {
- this.compressionType = compressionType;
- }
-
- @Field
- public void setConnectionsMaxIdleMs(int connectionsMaxIdleMs) {
- this.connectionsMaxIdleMs = connectionsMaxIdleMs;
- }
-
- @Field
- public void setDeliveryTimeoutMs(int deliveryTimeoutMs) {
- this.deliveryTimeoutMs = deliveryTimeoutMs;
- }
-
- @Field
- public void setEnableIdempotence(boolean enableIdempotence) {
- this.enableIdempotence = enableIdempotence;
- }
-
- @Field
- public void setInterceptorClasses(String interceptorClasses) {
- this.interceptorClasses = interceptorClasses;
- }
-
- @Field
- public void setMaxBlockMs(int maxBlockMs) {
- this.maxBlockMs = maxBlockMs;
- }
-
- @Field
- public void setMaxInFlightRequestsPerConnection(int maxInFlightRequestsPerConnection) {
- this.maxInFlightRequestsPerConnection = maxInFlightRequestsPerConnection;
- }
-
- @Field
- public void setMaxRequestSize(int maxRequestSize) {
- this.maxRequestSize = maxRequestSize;
- }
-
- @Field
- public void setMetadataMaxAgeMs(int metadataMaxAgeMs) {
- this.metadataMaxAgeMs = metadataMaxAgeMs;
- }
-
- @Field
- public void setRequestTimeoutMs(int requestTimeoutMs) {
- this.requestTimeoutMs = requestTimeoutMs;
- }
-
- @Field
- public void setRetries(int retries) {
- this.retries = retries;
- }
-
- @Field
- public void setRetryBackoffMs(int retryBackoffMs) {
- this.retryBackoffMs = retryBackoffMs;
- }
-
- @Field
- public void setTransactionTimeoutMs(int transactionTimeoutMs) {
- this.transactionTimeoutMs = transactionTimeoutMs;
- }
-
- @Field
- public void setTransactionalId(String transactionalId) {
- this.transactionalId = transactionalId;
- }
-
- @Field
- public void setKeySerializer(String keySerializer) {
- this.keySerializer = keySerializer;
- }
-
- @Field
- public void setValueSerializer(String valueSerializer) {
- this.valueSerializer = valueSerializer;
- }
-
- @Field
- public void setTopic(String topic) {
- this.topic = topic;
- }
-
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext)
- throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.isEmpty()) {
- throw new TikaEmitterException("metadata list must not be null or of size 0");
- }
- for (Metadata metadata : metadataList) {
- LOGGER.debug("about to emit to target topic: ({}) path:({})", topic, emitKey);
-
- Map<String, Object> fields = new HashMap<>();
- for (String n : metadata.names()) {
- String[] vals = metadata.getValues(n);
- if (vals.length > 1) {
- LOGGER.warn("Can only write the first value for key {}. I see {} values.",
- n,
- vals.length);
- }
- fields.put(n, vals[0]);
- }
-
- producer.send(new ProducerRecord<>(topic, emitKey, OM.writeValueAsString(fields)));
- }
- }
-
- private void safePut(Properties props, String key, Object val) {
- if (val != null) {
- props.put(key, val);
- }
- }
-
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
-
- // create instance for properties to access producer configs
- Properties props = new Properties();
-
- //Assign localhost id
- safePut(props, ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
-
- //Set acknowledgements for producer requests.
- safePut(props, ProducerConfig.ACKS_CONFIG, acks);
-
- //If the request fails, the producer can automatically retry,
- safePut(props, ProducerConfig.RETRIES_CONFIG, retries);
-
- //Specify buffer size in config
- safePut(props, ProducerConfig.BATCH_SIZE_CONFIG, batchSize);
-
- //Reduce the no of requests less than 0
- safePut(props, ProducerConfig.LINGER_MS_CONFIG, lingerMs);
-
- //The buffer.memory controls the total amount of memory available to the producer for buffering.
- safePut(props, ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory);
-
- safePut(props, ProducerConfig.CLIENT_ID_CONFIG, clientId);
- safePut(props, ProducerConfig.COMPRESSION_TYPE_CONFIG, compressionType);
- safePut(props, ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, deliveryTimeoutMs);
- safePut(props, ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, enableIdempotence);
- safePut(props, ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, interceptorClasses);
- safePut(props, ProducerConfig.MAX_BLOCK_MS_CONFIG, maxBlockMs);
- safePut(props, ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, maxInFlightRequestsPerConnection);
- safePut(props, ProducerConfig.MAX_REQUEST_SIZE_CONFIG, maxRequestSize);
- safePut(props, ProducerConfig.METADATA_MAX_AGE_CONFIG, metadataMaxAgeMs);
- safePut(props, ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, requestTimeoutMs);
- safePut(props, ProducerConfig.RETRY_BACKOFF_MS_CONFIG, retryBackoffMs);
- safePut(props, ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, transactionTimeoutMs);
- safePut(props, ProducerConfig.TRANSACTIONAL_ID_CONFIG, transactionalId);
-
- safePut(props, ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
- serializerClass(keySerializer, StringSerializer.class));
- safePut(props, ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
- serializerClass(valueSerializer, StringSerializer.class));
-
- producer = new KafkaProducer<>(props);
- }
-
- private Object serializerClass(String className, Class defaultClass) {
- try {
- return className == null ? defaultClass : Class.forName(className);
- } catch (ClassNotFoundException e) {
- LOGGER.error("Could not find key serializer class: {}", className);
- return null;
- }
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler)
- throws TikaConfigException {
- mustNotBeEmpty("topic", this.topic);
- mustNotBeEmpty("server", this.bootstrapServers);
- }
-
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/pom.xml b/tika-pipes/tika-emitters/tika-emitter-opensearch/pom.xml
deleted file mode 100644
index 7853eceb9..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/pom.xml
+++ /dev/null
@@ -1,135 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-opensearch</artifactId>
- <name>Apache Tika OpenSearch emitter</name>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-httpclient-commons</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-serialization</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-io</artifactId>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-http</artifactId>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.emitter.opensearch</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/JsonResponse.java b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/JsonResponse.java
deleted file mode 100644
index 9a5d36407..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/JsonResponse.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.opensearch;
-
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-public class JsonResponse {
-
- private final int status;
- private final String msg;
- private final JsonNode root;
-
- public JsonResponse(int status, JsonNode root) {
- this.status = status;
- this.root = root;
- this.msg = null;
- }
-
- public JsonResponse(int status, String msg) {
- this.status = status;
- this.msg = msg;
- this.root = null;
- }
-
- public int getStatus() {
- return status;
- }
-
- public String getMsg() {
- return msg;
- }
-
- public JsonNode getJson() {
- return root;
- }
-
- @Override
- public String toString() {
- return "JsonResponse{" +
- "status=" + status +
- ", msg='" + msg + '\'' +
- ", root=" + root +
- '}';
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClient.java b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClient.java
deleted file mode 100644
index d26ac46e0..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClient.java
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.opensearch;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.io.StringWriter;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.UUID;
-
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.client.TikaClientException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.pipes.core.emitter.EmitData;
-import org.apache.tika.utils.StringUtils;
-
-public class OpenSearchClient {
-
- private static final Logger LOG = LoggerFactory.getLogger(OpenSearchEmitter.class);
-
- //this includes the full url and the index, should not end in /
- //e.g. https://localhost:9200/my-index
- protected final String openSearchUrl;
- protected final HttpClient httpClient;
- private final OpenSearchEmitter.AttachmentStrategy attachmentStrategy;
-
- private final MetadataToJsonWriter metadataToJsonWriter;
- private final String embeddedFileFieldName;
- protected OpenSearchClient(String openSearchUrl, HttpClient httpClient,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy,
- OpenSearchEmitter.UpdateStrategy updateStrategy,
- String embeddedFileFieldName) {
- this.openSearchUrl = openSearchUrl;
- this.httpClient = httpClient;
- this.attachmentStrategy = attachmentStrategy;
- this.metadataToJsonWriter = (updateStrategy == OpenSearchEmitter.UpdateStrategy.OVERWRITE) ?
- new InsertMetadataToJsonWriter() : new UpsertMetadataToJsonWriter();
- this.embeddedFileFieldName = embeddedFileFieldName;
- }
-
-
- public void emitDocuments(List<? extends EmitData> emitData) throws IOException, TikaClientException {
- StringBuilder json = new StringBuilder();
- for (EmitData d : emitData) {
- appendDoc(d.getEmitKey().getEmitKey(), d.getMetadataList(), json);
- }
- emitJson(json);
- }
-
- private void emitJson(StringBuilder json) throws IOException, TikaClientException {
- String requestUrl = openSearchUrl + "/_bulk";
- JsonResponse response = postJson(requestUrl, json.toString());
- if (response.getStatus() != 200) {
- throw new TikaClientException(response.getMsg());
- } else {
- //if there's a single error, throw the full json.
- //this has not been thoroughly tested with versions of es < 7
- JsonNode errorNode = response.getJson().get("errors");
- if (errorNode.asText().equals("true")) {
- throw new TikaClientException(response.getJson().toString());
- }
- }
- }
-
-
- public void emitDocument(String emitKey, List<Metadata> metadataList) throws IOException,
- TikaClientException {
-
- StringBuilder json = new StringBuilder();
- appendDoc(emitKey, metadataList, json);
- emitJson(json);
- }
-
- private void appendDoc(String emitKey, List<Metadata> metadataList, StringBuilder json)
- throws IOException {
- int i = 0;
- String routing = (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.PARENT_CHILD) ?
- emitKey : null;
-
- for (Metadata metadata : metadataList) {
- StringBuilder id = new StringBuilder(emitKey);
- if (i > 0) {
- id.append("-").append(UUID.randomUUID());
- }
- String indexJson = metadataToJsonWriter.getBulkJson(id.toString(), routing);
- json.append(indexJson).append("\n");
- if (i == 0) {
- json.append(metadataToJsonWriter.writeContainer(metadata, attachmentStrategy));
- } else {
- json.append(metadataToJsonWriter.writeEmbedded(metadata, attachmentStrategy, emitKey,
- embeddedFileFieldName));
- }
- json.append("\n");
- i++;
- }
- }
-
- //Only here for testing. These may disappear without notice in the future.
- protected static String metadataToJsonContainerInsert(Metadata metadata,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy)
- throws IOException {
- return new InsertMetadataToJsonWriter().writeContainer(metadata, attachmentStrategy);
- }
-
- //Only here for testing. These may disappear without notice in the future.
- protected static String metadataToJsonEmbeddedInsert(Metadata metadata,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy,
- String emitKey, String embeddedFileFieldName)
- throws IOException {
- return new InsertMetadataToJsonWriter().writeEmbedded(metadata,
- attachmentStrategy, emitKey, embeddedFileFieldName);
- }
-
- public JsonResponse postJson(String url, String json) throws IOException {
- HttpPost httpRequest = new HttpPost(url);
- StringEntity entity = new StringEntity(json, StandardCharsets.UTF_8);
- httpRequest.setEntity(entity);
- httpRequest.setHeader("Accept", "application/json");
- httpRequest.setHeader("Content-type", "application/json; charset=utf-8");
- //At one point, this was required because of connection already
- // bound exceptions on windows :(
- //httpPost.setHeader("Connection", "close");
-
- //try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
-
- HttpResponse response = null;
- try {
- response = httpClient.execute(httpRequest);
- int status = response.getStatusLine().getStatusCode();
- if (status == 200) {
- try (Reader reader = new BufferedReader(
- new InputStreamReader(response.getEntity().getContent(),
- StandardCharsets.UTF_8))) {
- ObjectMapper mapper = new ObjectMapper();
- JsonNode node = mapper.readTree(reader);
- if (LOG.isTraceEnabled()) {
- LOG.trace("node: {}", node);
- }
- return new JsonResponse(200, node);
- }
- } else {
- return new JsonResponse(status,
- new String(EntityUtils.toByteArray(response.getEntity()),
- StandardCharsets.UTF_8));
- }
- } finally {
- if (response != null && response instanceof CloseableHttpResponse) {
- ((CloseableHttpResponse)response).close();
- }
- httpRequest.releaseConnection();
- }
- }
-
- private interface MetadataToJsonWriter {
- String writeContainer(Metadata metadata, OpenSearchEmitter.AttachmentStrategy attachmentStrategy)
- throws IOException;
-
- String writeEmbedded(Metadata metadata, OpenSearchEmitter.AttachmentStrategy attachmentStrategy,
- String emitKey, String embeddedFileFieldName) throws IOException;
-
- String getBulkJson(String id, String routing) throws IOException;
- }
-
- private static class InsertMetadataToJsonWriter implements MetadataToJsonWriter {
-
- @Override
- public String writeContainer(Metadata metadata,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy)
- throws IOException {
- StringWriter writer = new StringWriter();
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
- jsonGenerator.writeStartObject();
- writeMetadata(metadata, jsonGenerator);
- if (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.PARENT_CHILD) {
- jsonGenerator.writeStringField("relation_type", "container");
- }
- jsonGenerator.writeEndObject();
- }
- return writer.toString();
- }
-
- @Override
- public String writeEmbedded(Metadata metadata,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy,
- String emitKey, String embeddedFileFieldName)
- throws IOException {
- StringWriter writer = new StringWriter();
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
- jsonGenerator.writeStartObject();
-
- writeMetadata(metadata, jsonGenerator);
- if (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.PARENT_CHILD) {
- jsonGenerator.writeObjectFieldStart("relation_type");
- jsonGenerator.writeStringField("name", embeddedFileFieldName);
- jsonGenerator.writeStringField("parent", emitKey);
- //end the relation type object
- jsonGenerator.writeEndObject();
- } else if (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.SEPARATE_DOCUMENTS) {
- jsonGenerator.writeStringField("parent", emitKey);
- }
- //end the metadata object
- jsonGenerator.writeEndObject();
- }
- return writer.toString();
- }
-
- @Override
- public String getBulkJson(String id, String routing) throws IOException {
- StringWriter writer = new StringWriter();
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
- jsonGenerator.writeStartObject();
- jsonGenerator.writeObjectFieldStart("index");
- jsonGenerator.writeStringField("_id", id);
- if (!StringUtils.isEmpty(routing)) {
- jsonGenerator.writeStringField("routing", routing);
- }
- jsonGenerator.writeEndObject();
- jsonGenerator.writeEndObject();
- }
- return writer.toString();
- }
- }
-
- private static class UpsertMetadataToJsonWriter implements MetadataToJsonWriter {
-
- @Override
- public String writeContainer(Metadata metadata,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy)
- throws IOException {
- StringWriter writer = new StringWriter();
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
- jsonGenerator.writeStartObject();
- jsonGenerator.writeObjectFieldStart("doc");
- writeMetadata(metadata, jsonGenerator);
- if (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.PARENT_CHILD) {
- jsonGenerator.writeStringField("relation_type", "container");
- }
- jsonGenerator.writeEndObject();
- jsonGenerator.writeBooleanField("doc_as_upsert", true);
- jsonGenerator.writeEndObject();
- }
- return writer.toString();
- }
-
- @Override
- public String writeEmbedded(Metadata metadata,
- OpenSearchEmitter.AttachmentStrategy attachmentStrategy,
- String emitKey, String embeddedFileFieldName)
- throws IOException {
- StringWriter writer = new StringWriter();
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
- jsonGenerator.writeStartObject();
- jsonGenerator.writeObjectFieldStart("doc");
- writeMetadata(metadata, jsonGenerator);
- if (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.PARENT_CHILD) {
- jsonGenerator.writeObjectFieldStart("relation_type");
- jsonGenerator.writeStringField("name", embeddedFileFieldName);
- jsonGenerator.writeStringField("parent", emitKey);
- //end the relation type object
- jsonGenerator.writeEndObject();
- } else if (attachmentStrategy == OpenSearchEmitter.AttachmentStrategy.SEPARATE_DOCUMENTS) {
- jsonGenerator.writeStringField("parent", emitKey);
- }
- //end the "doc"
- jsonGenerator.writeEndObject();
- jsonGenerator.writeBooleanField("doc_as_upsert", true);
- //end the metadata object
- jsonGenerator.writeEndObject();
- }
- return writer.toString();
- }
-
- @Override
- public String getBulkJson(String id, String routing) throws IOException {
- StringWriter writer = new StringWriter();
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
- jsonGenerator.writeStartObject();
- jsonGenerator.writeObjectFieldStart("update");
- jsonGenerator.writeStringField("_id", id);
- if (!StringUtils.isEmpty(routing)) {
- jsonGenerator.writeStringField("routing", routing);
- }
- jsonGenerator.writeNumberField("retry_on_conflict", 3);
- jsonGenerator.writeEndObject();
- jsonGenerator.writeEndObject();
- }
- return writer.toString();
- }
- }
-
- private static void writeMetadata(Metadata metadata, JsonGenerator jsonGenerator) throws IOException {
- //writes the metadata without the start { or the end }
- //to allow for other fields to be added
- for (String n : metadata.names()) {
- String[] vals = metadata.getValues(n);
- if (vals.length == 1) {
- jsonGenerator.writeStringField(n, vals[0]);
- } else {
- jsonGenerator.writeArrayFieldStart(n);
- for (String v : vals) {
- jsonGenerator.writeString(v);
- }
- jsonGenerator.writeEndArray();
- }
- }
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchEmitter.java b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchEmitter.java
deleted file mode 100644
index 3b8c21b8d..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchEmitter.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.opensearch;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.client.HttpClientFactory;
-import org.apache.tika.client.TikaClientException;
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.EmitData;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-import org.apache.tika.utils.StringUtils;
-
-
-public class OpenSearchEmitter extends AbstractEmitter implements Initializable {
-
-
- public enum AttachmentStrategy {
- SEPARATE_DOCUMENTS, PARENT_CHILD,
- //anything else?
- }
-
- public enum UpdateStrategy {
- OVERWRITE, UPSERT
- //others?
- }
-
- public static String DEFAULT_EMBEDDED_FILE_FIELD_NAME = "embedded";
- private static final Logger LOG = LoggerFactory.getLogger(OpenSearchEmitter.class);
- private AttachmentStrategy attachmentStrategy = AttachmentStrategy.PARENT_CHILD;
-
- private UpdateStrategy updateStrategy = UpdateStrategy.OVERWRITE;
- private String openSearchUrl = null;
- private String idField = "_id";
- private int commitWithin = 1000;
- private OpenSearchClient openSearchClient;
- private final HttpClientFactory httpClientFactory;
- private String embeddedFileFieldName = DEFAULT_EMBEDDED_FILE_FIELD_NAME;
-
- public OpenSearchEmitter() throws TikaConfigException {
- httpClientFactory = new HttpClientFactory();
- }
-
- @Override
- public void emit(List<? extends EmitData> emitData) throws IOException, TikaEmitterException {
- if (emitData == null || emitData.size() == 0) {
- LOG.debug("metadataList is null or empty");
- return;
- }
- try {
- LOG.debug("about to emit {} docs", emitData.size());
- openSearchClient.emitDocuments(emitData);
- LOG.info("successfully emitted {} docs", emitData.size());
- } catch (TikaClientException e) {
- LOG.warn("problem emitting docs", e);
- throw new TikaEmitterException(e.getMessage());
- }
- }
-
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext)
- throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.size() == 0) {
- LOG.debug("metadataList is null or empty");
- return;
- }
- try {
- LOG.debug("about to emit one doc");
- openSearchClient.emitDocument(emitKey, metadataList);
- LOG.info("successfully emitted one doc");
- } catch (TikaClientException e) {
- LOG.warn("problem emitting doc", e);
- throw new TikaEmitterException("failed to add document", e);
- }
- }
-
-
- /**
- * Options: SEPARATE_DOCUMENTS, PARENT_CHILD. Default is "SEPARATE_DOCUMENTS".
- * All embedded documents are treated as independent documents.
- * PARENT_CHILD requires a schema to be set up for the relationship type;
- * all embedded objects (no matter how deeply nested) will have a single
- * parent of the main container document.
- *
- * If you want to concatenate the content of embedded files and ignore
- * the metadata of embedded files, set
- * {@link org.apache.tika.pipes.HandlerConfig}'s parseMode to
- * {@link org.apache.tika.pipes.HandlerConfig.PARSE_MODE#CONCATENATE}
- * in your {@link org.apache.tika.pipes.FetchEmitTuple} or in the
- * <parseMode> element in your {@link org.apache.tika.pipes.pipesiterator.PipesIterator}
- * configuration.
- */
- @Field
- public void setAttachmentStrategy(String attachmentStrategy) {
- this.attachmentStrategy = AttachmentStrategy.valueOf(attachmentStrategy);
- }
-
-
- @Field
- public void setConnectionTimeout(int connectionTimeout) {
- httpClientFactory.setConnectTimeout(connectionTimeout);
- }
-
- @Field
- public void setSocketTimeout(int socketTimeout) {
- httpClientFactory.setSocketTimeout(socketTimeout);
- }
-
- public int getCommitWithin() {
- return commitWithin;
- }
-
- @Field
- public void setCommitWithin(int commitWithin) {
- this.commitWithin = commitWithin;
- }
-
- /**
- * Specify the field in the first Metadata that should be
- * used as the id field for the document.
- *
- * @param idField
- */
- @Field
- public void setIdField(String idField) {
- this.idField = idField;
- }
-
- //this is the full url, including the collection, e.g. https://localhost:9200/my-collection
- @Field
- public void setOpenSearchUrl(String openSearchUrl) {
- this.openSearchUrl = openSearchUrl;
- }
-
- //TODO -- add other httpclient configurations??
- @Field
- public void setUserName(String userName) {
- httpClientFactory.setUserName(userName);
- }
-
- @Field
- public void setPassword(String password) {
- httpClientFactory.setPassword(password);
- }
-
- @Field
- public void setAuthScheme(String authScheme) {
- httpClientFactory.setAuthScheme(authScheme);
- }
-
- @Field
- public void setProxyHost(String proxyHost) {
- httpClientFactory.setProxyHost(proxyHost);
- }
-
- @Field
- public void setProxyPort(int proxyPort) {
- httpClientFactory.setProxyPort(proxyPort);
- }
-
- public void setUpdateStrategy(UpdateStrategy updateStrategy) {
- this.updateStrategy = updateStrategy;
- }
-
- public void setUpdateStrategy(String strategy) throws TikaConfigException {
- switch (strategy.toLowerCase(Locale.US)) {
- case "overwrite" :
- setUpdateStrategy(UpdateStrategy.OVERWRITE);
- break;
- case "upsert" :
- setUpdateStrategy(UpdateStrategy.UPSERT);
- break;
- default :
- throw new TikaConfigException("'overwrite' and 'upsert' are the two options so " +
- "far. I regret I don't understand: " + strategy);
- }
- }
-
- /**
- * If using the {@link AttachmentStrategy#PARENT_CHILD}, this is the field name
- * used to store the child documents. Note that we artificially flatten all embedded
- * documents, no matter how nested in the container document, into direct children
- * of the root document.
- *
- * @param embeddedFileFieldName
- */
- @Field
- public void setEmbeddedFileFieldName(String embeddedFileFieldName) {
- this.embeddedFileFieldName = embeddedFileFieldName;
- }
-
-
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- if (StringUtils.isBlank(openSearchUrl)) {
- throw new TikaConfigException("Must specify an open search url!");
- } else {
- openSearchClient =
- new OpenSearchClient(openSearchUrl,
- httpClientFactory.build(), attachmentStrategy, updateStrategy,
- embeddedFileFieldName);
- }
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler)
- throws TikaConfigException {
- mustNotBeEmpty("openSearchUrl", this.openSearchUrl);
- mustNotBeEmpty("idField", this.idField);
- }
-
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClientTest.java b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClientTest.java
deleted file mode 100644
index 994ff0044..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClientTest.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.opensearch;
-
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.TikaTest;
-import org.apache.tika.metadata.Metadata;
-
-public class OpenSearchClientTest extends TikaTest {
-
- @Test
- public void testSerialization() throws Exception {
- Metadata metadata = new Metadata();
- metadata.add("authors", "author1");
- metadata.add("authors", "author2");
- metadata.add("title", "title1");
- for (OpenSearchEmitter.AttachmentStrategy strategy :
- OpenSearchEmitter.AttachmentStrategy.values()) {
- String json = OpenSearchClient.metadataToJsonContainerInsert(metadata,
- strategy);
- assertContains("author1", json);
- assertContains("author2", json);
- assertContains("authors", json);
- assertContains("title1", json);
- }
- for (OpenSearchEmitter.AttachmentStrategy strategy :
- OpenSearchEmitter.AttachmentStrategy.values()) {
- String json = OpenSearchClient.metadataToJsonEmbeddedInsert(metadata, strategy,
- "myEmitKey", OpenSearchEmitter.DEFAULT_EMBEDDED_FILE_FIELD_NAME);
- assertContains("author1", json);
- assertContains("author2", json);
- assertContains("authors", json);
- assertContains("title1", json);
- }
-
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/resources/tika-config-simple-emitter.xml b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/resources/tika-config-simple-emitter.xml
deleted file mode 100644
index f6530a9e9..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/resources/tika-config-simple-emitter.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <metadataFilters>
- <metadataFilter class="org.apache.tika.metadata.filter.FieldNameMappingFilter">
- <mappings>
- <mapping from="X-TIKA:content" to="content"/>
- <mapping from="X-TIKA:embedded_resource_path" to="embedded_path"/>
- <mapping from="Content-Length" to="length"/>
- <mapping from="dc:creator" to="creators"/>
- <mapping from="dc:title" to="title"/>
- </mappings>
- </metadataFilter>
- </metadataFilters>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.opensearch.OpenSearchEmitter">
- <name>opensearch1</name>
- <url>http://localhost:9200/tika-test</url>
- <attachmentStrategy>concatenate-content</attachmentStrategy>
- <contentField>content</contentField>
- <commitWithin>10</commitWithin>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml b/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml
deleted file mode 100644
index b48a8c302..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml
+++ /dev/null
@@ -1,112 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-s3</artifactId>
- <name>Apache Tika S3 emitter</name>
-
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.emitter.s3</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java b/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
deleted file mode 100644
index c454c6c6a..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.s3;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-import java.util.List;
-import java.util.Map;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.InstanceProfileCredentialsProvider;
-import com.amazonaws.auth.profile.ProfileCredentialsProvider;
-import com.amazonaws.client.builder.AwsClientBuilder;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TemporaryResources;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.StreamEmitter;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-import org.apache.tika.serialization.JsonMetadataList;
-import org.apache.tika.utils.StringUtils;
-
-/**
- * Emits to existing s3 bucket
- * <pre class="prettyprint">
- * <properties>
- * <emitters>
- * <emitter class="org.apache.tika.pipes.emitter.s3.S3Emitter>
- * <params>
- * <!-- required -->
- * <param name="name" type="string">s3e</param>
- * <!-- required -->
- * <param name="region" type="string">us-east-1</param>
- * <!-- required -->
- * <param name="credentialsProvider"
- * type="string">(profile|instance)</param>
- * <!-- required if credentialsProvider=profile-->
- * <param name="profile" type="string">my-profile</param>
- * <!-- required -->
- * <param name="bucket" type="string">my-bucket</param>
- * <!-- optional; prefix to add to the path before emitting;
- * default is no prefix -->
- * <param name="prefix" type="string">my-prefix</param>
- * <!-- optional; default is 'json' this will be added to the SOURCE_PATH
- * if no emitter key is specified. Do not add a "."
- * before the extension -->
- * <param name="fileExtension" type="string">json</param>
- * <!-- optional; default is 'true'-- whether to copy the
- * json to a local file before putting to s3 -->
- * <param name="spoolToTemp" type="bool">true</param>
- * </params>
- * </emitter>
- * </emitters>
- * </properties></pre>
- */
-public class S3Emitter extends AbstractEmitter implements Initializable, StreamEmitter {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(S3Emitter.class);
- private String region;
- private String profile;
- private String bucket;
- private String credentialsProvider;
- private String accessKey;
- private String secretKey;
- private String endpointConfigurationService;
- private String fileExtension = "json";
- private boolean spoolToTemp = true;
- private String prefix = null;
- private int maxConnections = ClientConfiguration.DEFAULT_MAX_CONNECTIONS;
- private boolean pathStyleAccessEnabled = false;
- private AmazonS3 s3Client;
-
- /**
- * Requires the src-bucket/path/to/my/file.txt in the {@link TikaCoreProperties#SOURCE_PATH}.
- *
- * @param metadataList
- * @throws IOException
- * @throws TikaException
- */
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext) throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.size() == 0) {
- throw new TikaEmitterException("metadata list must not be null or of size 0");
- }
-
- if (!spoolToTemp) {
- UnsynchronizedByteArrayOutputStream bos = UnsynchronizedByteArrayOutputStream
- .builder()
- .get();
- try (Writer writer = new BufferedWriter(new OutputStreamWriter(bos, StandardCharsets.UTF_8))) {
- JsonMetadataList.toJson(metadataList, writer);
- } catch (IOException e) {
- throw new TikaEmitterException("can't jsonify", e);
- }
- byte[] bytes = bos.toByteArray();
- try (InputStream is = TikaInputStream.get(bytes)) {
- emit(emitKey, is, new Metadata(), parseContext);
- }
- } else {
- try (TemporaryResources tmp = new TemporaryResources()) {
- Path tmpPath = tmp.createTempFile();
- try (Writer writer = Files.newBufferedWriter(tmpPath, StandardCharsets.UTF_8, StandardOpenOption.CREATE)) {
- JsonMetadataList.toJson(metadataList, writer);
- } catch (IOException e) {
- throw new TikaEmitterException("can't jsonify", e);
- }
- try (InputStream is = TikaInputStream.get(tmpPath)) {
- emit(emitKey, is, new Metadata(), parseContext);
- }
- }
- }
- }
-
- /**
- * @param path -- object path, not including the bucket
- * @param is inputStream to copy
- * @param userMetadata this will be written to the s3 ObjectMetadata's userMetadata
- * @throws TikaEmitterException or IOexception if there is a Runtime s3 client exception
- */
- @Override
- public void emit(String path, InputStream is, Metadata userMetadata, ParseContext parseContext) throws IOException, TikaEmitterException {
-
- if (!StringUtils.isBlank(prefix)) {
- path = prefix + "/" + path;
- }
-
- if (!StringUtils.isBlank(fileExtension)) {
- path += "." + fileExtension;
- }
-
- LOGGER.debug("about to emit to target bucket: ({}) path:({})", bucket, path);
-
- ObjectMetadata objectMetadata = new ObjectMetadata();
- for (String n : userMetadata.names()) {
- String[] vals = userMetadata.getValues(n);
- if (vals.length > 1) {
- LOGGER.warn("Can only write the first value for key {}. I see {} values.", n, vals.length);
- }
- objectMetadata.addUserMetadata(n, vals[0]);
- }
- //In practice, sending a file is more robust
- //We ran into stream reset issues during digesting, and aws doesn't
- //like putObjects for streams without lengths
- if (is instanceof TikaInputStream) {
- if (((TikaInputStream) is).hasFile()) {
- try {
- PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, path, ((TikaInputStream) is).getFile()).withMetadata(objectMetadata);
- s3Client.putObject(putObjectRequest);
- } catch (IOException e) {
- throw new TikaEmitterException("exception sending underlying file", e);
- }
- return;
- }
- }
- try {
- s3Client.putObject(bucket, path, is, objectMetadata);
- } catch (AmazonClientException e) {
- throw new IOException("problem writing s3object", e);
- }
- }
-
- /**
- * Whether or not to spool the metadatalist to a tmp file before putting object.
- * Default: <code>true</code>. If this is set to <code>false</code>,
- * this emitter writes the json object to memory and then puts that into s3.
- *
- * @param spoolToTemp
- */
- @Field
- public void setSpoolToTemp(boolean spoolToTemp) {
- this.spoolToTemp = spoolToTemp;
- }
-
- @Field
- public void setRegion(String region) {
- this.region = region;
- }
-
- @Field
- public void setProfile(String profile) {
- this.profile = profile;
- }
-
- @Field
- public void setBucket(String bucket) {
- this.bucket = bucket;
- }
-
- @Field
- public void setPrefix(String prefix) {
- //strip final "/" if it exists
- if (prefix.endsWith("/")) {
- this.prefix = prefix.substring(0, prefix.length() - 1);
- } else {
- this.prefix = prefix;
- }
- }
-
- @Field
- public void setCredentialsProvider(String credentialsProvider) {
- if (!credentialsProvider.equals("profile") && !credentialsProvider.equals("instance") && !credentialsProvider.equals("key_secret")) {
- throw new IllegalArgumentException("credentialsProvider must be either 'profile', 'instance' or 'key_secret'");
- }
- this.credentialsProvider = credentialsProvider;
- }
-
- /**
- * If you want to customize the output file's file extension.
- * Do not include the "."
- *
- * @param fileExtension
- */
- @Field
- public void setFileExtension(String fileExtension) {
- this.fileExtension = fileExtension;
- }
-
- @Field
- public void setAccessKey(String accessKey) {
- this.accessKey = accessKey;
- }
-
- @Field
- public void setSecretKey(String secretKey) {
- this.secretKey = secretKey;
- }
-
- /**
- * maximum number of http connections allowed. This should be
- * greater than or equal to the number of threads emitting to S3.
- *
- * @param maxConnections
- */
- @Field
- public void setMaxConnections(int maxConnections) {
- this.maxConnections = maxConnections;
- }
-
- @Field
- public void setEndpointConfigurationService(String endpointConfigurationService) {
- this.endpointConfigurationService = endpointConfigurationService;
- }
-
- /**
- * This initializes the s3 client. Note, we wrap S3's RuntimeExceptions,
- * e.g. AmazonClientException in a TikaConfigException.
- *
- * @param params params to use for initialization
- * @throws TikaConfigException
- */
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- //params have already been set...ignore them
- AWSCredentialsProvider provider;
- if ("instance".equals(credentialsProvider)) {
- provider = InstanceProfileCredentialsProvider.getInstance();
- } else if ("profile".equals(credentialsProvider)) {
- provider = new ProfileCredentialsProvider(profile);
- } else if (credentialsProvider.equals("key_secret")) {
- provider = new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey));
- } else {
- throw new TikaConfigException("credentialsProvider must be set and " + "must be either 'instance', 'profile' or 'key_secret'");
- }
- ClientConfiguration clientConfig = new ClientConfiguration().withMaxConnections(maxConnections);
- try {
- AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder
- .standard()
- .withClientConfiguration(clientConfig)
- .withCredentials(provider)
- .withPathStyleAccessEnabled(pathStyleAccessEnabled);
- if (!StringUtils.isBlank(endpointConfigurationService)) {
- amazonS3ClientBuilder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpointConfigurationService, region));
- } else {
- amazonS3ClientBuilder.withRegion(region);
- }
- s3Client = amazonS3ClientBuilder.build();
- } catch (AmazonClientException e) {
- throw new TikaConfigException("can't initialize s3 emitter", e);
- }
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
- mustNotBeEmpty("bucket", this.bucket);
- mustNotBeEmpty("region", this.region);
- }
-
- @Field
- public void setPathStyleAccessEnabled(boolean pathStyleAccessEnabled) {
- this.pathStyleAccessEnabled = pathStyleAccessEnabled;
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml b/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
deleted file mode 100644
index b74dd0a67..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
+++ /dev/null
@@ -1,117 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-emitters</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-emitter-solr</artifactId>
- <name>Apache Tika Apache Solr emitter</name>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-httpclient-commons</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-solrj</artifactId>
- <version>${solrj.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.emitter.solr</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java
deleted file mode 100644
index 7f46304d5..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.solr;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.Http2SolrClient;
-import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.response.UpdateResponse;
-import org.apache.solr.common.SolrInputDocument;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.client.HttpClientFactory;
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.AbstractEmitter;
-import org.apache.tika.pipes.core.emitter.EmitData;
-import org.apache.tika.pipes.core.emitter.TikaEmitterException;
-import org.apache.tika.utils.StringUtils;
-
-
-public class SolrEmitter extends AbstractEmitter implements Initializable {
-
- public static String DEFAULT_EMBEDDED_FILE_FIELD_NAME = "embedded";
- private static final Logger LOG = LoggerFactory.getLogger(SolrEmitter.class);
- private final HttpClientFactory httpClientFactory;
- private AttachmentStrategy attachmentStrategy = AttachmentStrategy.PARENT_CHILD;
- private UpdateStrategy updateStrategy = UpdateStrategy.ADD;
- private String solrCollection;
- /**
- * You can specify solrUrls, or you can specify solrZkHosts and use use zookeeper to determine the solr server urls.
- */
- private List<String> solrUrls;
- private List<String> solrZkHosts;
- private String solrZkChroot;
- private String idField = "id";
- private int commitWithin = 1000;
- private int connectionTimeout = 10000;
- private int socketTimeout = 60000;
- private SolrClient solrClient;
- private String embeddedFileFieldName = DEFAULT_EMBEDDED_FILE_FIELD_NAME;
-
- public SolrEmitter() throws TikaConfigException {
- httpClientFactory = new HttpClientFactory();
- }
-
- @Override
- public void emit(String emitKey, List<Metadata> metadataList, ParseContext parseContext)
- throws IOException, TikaEmitterException {
- if (metadataList == null || metadataList.size() == 0) {
- LOG.warn("metadataList is null or empty");
- return;
- }
- List<SolrInputDocument> docsToUpdate = new ArrayList<>();
- addMetadataAsSolrInputDocuments(emitKey, metadataList, docsToUpdate);
- emitSolrBatch(docsToUpdate);
- }
-
- private void addMetadataAsSolrInputDocuments(String emitKey, List<Metadata> metadataList,
- List<SolrInputDocument> docsToUpdate)
- throws IOException, TikaEmitterException {
- SolrInputDocument solrInputDocument = new SolrInputDocument();
- solrInputDocument.setField(idField, emitKey);
- if (updateStrategy == UpdateStrategy.UPDATE_MUST_EXIST) {
- solrInputDocument.setField("_version_", 1);
- } else if (updateStrategy == UpdateStrategy.UPDATE_MUST_NOT_EXIST) {
- solrInputDocument.setField("_version_", -1);
- }
- if (metadataList.size() == 1) {
- addMetadataToSolrInputDocument(metadataList.get(0), solrInputDocument, updateStrategy);
- docsToUpdate.add(solrInputDocument);
- } else if (attachmentStrategy == AttachmentStrategy.PARENT_CHILD) {
- addMetadataToSolrInputDocument(metadataList.get(0), solrInputDocument, updateStrategy);
- List<SolrInputDocument> children = new ArrayList<>();
- for (int i = 1; i < metadataList.size(); i++) {
- SolrInputDocument childSolrInputDocument = new SolrInputDocument();
- Metadata m = metadataList.get(i);
- childSolrInputDocument
- .setField(idField, emitKey + "-" + UUID.randomUUID().toString());
- addMetadataToSolrInputDocument(m, childSolrInputDocument, updateStrategy);
- children.add(childSolrInputDocument);
- }
- solrInputDocument.setField(embeddedFileFieldName, children);
- docsToUpdate.add(solrInputDocument);
- } else if (attachmentStrategy == AttachmentStrategy.SEPARATE_DOCUMENTS) {
- addMetadataToSolrInputDocument(metadataList.get(0), solrInputDocument, updateStrategy);
- docsToUpdate.add(solrInputDocument);
- for (int i = 1; i < metadataList.size(); i++) {
- SolrInputDocument childSolrInputDocument = new SolrInputDocument();
- Metadata m = metadataList.get(i);
- childSolrInputDocument.setField(idField,
- solrInputDocument.get(idField).getValue() + "-" + UUID.randomUUID().toString());
- addMetadataToSolrInputDocument(m, childSolrInputDocument, updateStrategy);
- docsToUpdate.add(childSolrInputDocument);
- }
- } else {
- throw new IllegalArgumentException(
- "I don't yet support this attachment strategy: " + attachmentStrategy);
- }
- }
-
- @Override
- public void emit(List<? extends EmitData> batch) throws IOException, TikaEmitterException {
- if (batch == null || batch.size() == 0) {
- LOG.warn("batch is null or empty");
- return;
- }
- List<SolrInputDocument> docsToUpdate = new ArrayList<>();
- for (EmitData d : batch) {
- addMetadataAsSolrInputDocuments(d.getEmitKey().getEmitKey(), d.getMetadataList(),
- docsToUpdate);
- }
- emitSolrBatch(docsToUpdate);
- }
-
- private void emitSolrBatch(List<SolrInputDocument> docsToUpdate)
- throws IOException, TikaEmitterException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Emitting solr doc batch: {}", docsToUpdate);
- }
- if (!docsToUpdate.isEmpty()) {
- try {
- UpdateRequest req = new UpdateRequest();
- req.add(docsToUpdate);
- req.setCommitWithin(commitWithin);
- req.setParam("failOnVersionConflicts", "false");
- UpdateResponse updateResponse = req.process(solrClient, solrCollection);
- LOG.debug("update response: " + updateResponse);
- if (updateResponse.getStatus() != 0) {
- throw new TikaEmitterException("Bad status: " + updateResponse);
- }
- } catch (Exception e) {
- throw new TikaEmitterException("Could not add batch to solr", e);
- }
- }
- }
-
- private void addMetadataToSolrInputDocument(Metadata metadata,
- SolrInputDocument solrInputDocument,
- UpdateStrategy updateStrategy) {
- for (String n : metadata.names()) {
- String[] vals = metadata.getValues(n);
- if (vals.length == 0) {
- continue;
- } else if (vals.length == 1) {
- if (updateStrategy == UpdateStrategy.ADD) {
- solrInputDocument.setField(n, vals[0]);
- } else {
- solrInputDocument.setField(n, new HashMap<String, String>() {
- {
- put("set", vals[0]);
- }
- });
- }
- } else if (vals.length > 1) {
- if (updateStrategy == UpdateStrategy.ADD) {
- solrInputDocument.setField(n, vals);
- } else {
- solrInputDocument.setField(n, new HashMap<String, String[]>() {
- {
- put("set", vals);
- }
- });
- }
- }
- }
- }
-
- /**
- * Options: SKIP, CONCATENATE_CONTENT, PARENT_CHILD. Default is "PARENT_CHILD".
- * If set to "SKIP", this will index only the main file and ignore all info
- * in the attachments. If set to "CONCATENATE_CONTENT", this will concatenate the
- * content extracted from the attachments into the main document and
- * then index the main document with the concatenated content _and_ the
- * main document's metadata (metadata from attachments will be thrown away).
- * If set to "PARENT_CHILD", this will index the attachments as children
- * of the parent document via Solr's parent-child relationship.
- */
- @Field
- public void setAttachmentStrategy(String attachmentStrategy) {
- this.attachmentStrategy = AttachmentStrategy.valueOf(attachmentStrategy);
- }
-
- @Field
- public void setUpdateStrategy(String updateStrategy) {
- this.updateStrategy = UpdateStrategy.valueOf(updateStrategy);
- }
-
- @Field
- public void setConnectionTimeout(int connectionTimeout) {
- this.connectionTimeout = connectionTimeout;
- }
-
- @Field
- public void setSocketTimeout(int socketTimeout) {
- this.socketTimeout = socketTimeout;
- }
-
- public int getCommitWithin() {
- return commitWithin;
- }
-
- @Field
- public void setCommitWithin(int commitWithin) {
- this.commitWithin = commitWithin;
- }
-
- /**
- * Specify the field in the first Metadata that should be
- * used as the id field for the document.
- *
- * @param idField
- */
- @Field
- public void setIdField(String idField) {
- this.idField = idField;
- }
-
- @Field
- public void setSolrCollection(String solrCollection) {
- this.solrCollection = solrCollection;
- }
-
- @Field
- public void setSolrUrls(List<String> solrUrls) {
- this.solrUrls = solrUrls;
- }
-
- @Field
- public void setSolrZkHosts(List<String> solrZkHosts) {
- this.solrZkHosts = solrZkHosts;
- }
-
- @Field
- public void setSolrZkChroot(String solrZkChroot) {
- this.solrZkChroot = solrZkChroot;
- }
-
- //TODO -- add other httpclient configurations??
- @Field
- public void setUserName(String userName) {
- httpClientFactory.setUserName(userName);
- }
-
- @Field
- public void setPassword(String password) {
- httpClientFactory.setPassword(password);
- }
-
- @Field
- public void setAuthScheme(String authScheme) {
- httpClientFactory.setAuthScheme(authScheme);
- }
-
- @Field
- public void setProxyHost(String proxyHost) {
- httpClientFactory.setProxyHost(proxyHost);
- }
-
- @Field
- public void setProxyPort(int proxyPort) {
- httpClientFactory.setProxyPort(proxyPort);
- }
-
- /**
- * If using the {@link AttachmentStrategy#PARENT_CHILD}, this is the field name
- * used to store the child documents. Note that we artificially flatten all embedded
- * documents, no matter how nested in the container document, into direct children
- * of the root document.
- *
- * @param embeddedFileFieldName
- */
- @Field
- public void setEmbeddedFileFieldName(String embeddedFileFieldName) {
- this.embeddedFileFieldName = embeddedFileFieldName;
- }
-
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- if (solrUrls == null || solrUrls.isEmpty()) {
- //TODO -- there's more that we need to pass through, including ssl etc.
- Http2SolrClient.Builder http2SolrClientBuilder = new Http2SolrClient.Builder();
- if (!StringUtils.isBlank(httpClientFactory.getUserName())) {
- http2SolrClientBuilder.withBasicAuthCredentials(httpClientFactory.getUserName(), httpClientFactory.getPassword());
- }
- http2SolrClientBuilder
- .withRequestTimeout(httpClientFactory.getRequestTimeout(), TimeUnit.MILLISECONDS)
- .withConnectionTimeout(connectionTimeout, TimeUnit.MILLISECONDS);
-
-
- Http2SolrClient http2SolrClient = http2SolrClientBuilder.build();
- solrClient = new CloudSolrClient.Builder(solrZkHosts, Optional.ofNullable(solrZkChroot))
- .withHttpClient(http2SolrClient)
- .build();
-
- } else {
- solrClient = new LBHttpSolrClient.Builder().withConnectionTimeout(connectionTimeout, TimeUnit.MILLISECONDS)
- .withSocketTimeout(socketTimeout, TimeUnit.MILLISECONDS).withHttpClient(httpClientFactory.build())
- .withBaseEndpoints(solrUrls.toArray(new String[]{})).build();
- }
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler)
- throws TikaConfigException {
- mustNotBeEmpty("solrCollection", this.solrCollection);
- mustNotBeEmpty("urlFieldName", this.idField);
- if ((this.solrUrls == null || this.solrUrls.isEmpty()) &&
- (this.solrZkHosts == null || this.solrZkHosts.isEmpty())) {
- throw new IllegalArgumentException(
- "expected either param solrUrls or param solrZkHosts, but neither was specified");
- }
- if (this.solrUrls != null && !this.solrUrls.isEmpty() && this.solrZkHosts != null &&
- !this.solrZkHosts.isEmpty()) {
- throw new IllegalArgumentException(
- "expected either param solrUrls or param solrZkHosts, but both were specified");
- }
- }
-
- public enum AttachmentStrategy {
- SEPARATE_DOCUMENTS, PARENT_CHILD,
- //anything else?
- }
-
- public enum UpdateStrategy {
- ADD, UPDATE_MUST_EXIST, UPDATE_MUST_NOT_EXIST,
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/SolrEmitterDevTest.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/SolrEmitterDevTest.java
deleted file mode 100644
index 1e68b1d4e..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/SolrEmitterDevTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.emitter.solr;
-
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.metadata.filter.FieldNameMappingFilter;
-import org.apache.tika.parser.ParseContext;
-
-/**
- * This is meant only for one off development tests with a locally
- * running instance of Solr. Please add unit tests to the
- * tika-integration-tests/tika-pipes-solr-integration-tests
- */
-@Disabled
-public class SolrEmitterDevTest {
-
- @Test
- public void oneOff() throws Exception {
- String core = "tika-example";
- String url = "http://localhost:8983/solr";
- String emitKey = "one-off-test-doc";
- SolrEmitter solrEmitter = new SolrEmitter();
- solrEmitter.setSolrUrls(Collections.singletonList(url));
- solrEmitter.setSolrCollection(core);
- solrEmitter.initialize(Collections.EMPTY_MAP);
-
- Metadata metadata = new Metadata();
- metadata.set(TikaCoreProperties.CREATED, new Date());
- metadata.set(TikaCoreProperties.TIKA_CONTENT, "the quick brown fox");
-
- Map<String, String> mappings = new HashMap();
- FieldNameMappingFilter filter = new FieldNameMappingFilter();
- mappings.put(TikaCoreProperties.CREATED.getName(), "created");
- mappings.put(TikaCoreProperties.TIKA_CONTENT.getName(), "content");
- filter.setMappings(mappings);
- filter.filter(metadata);
-
- solrEmitter.emit(emitKey, Collections.singletonList(metadata), new ParseContext());
- }
-}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j2.xml b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j2.xml
deleted file mode 100644
index 5611a3556..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/log4j2.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<Configuration status="WARN">
- <Appenders>
- <Console name="Console" target="SYSTEM_ERR">
- <PatternLayout pattern="%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n"/>
- </Console>
- </Appenders>
- <Loggers>
- <Root level="debug">
- <AppenderRef ref="Console"/>
- </Root>
- <Logger name="org.apache.http" level="INFO" additivity="false">
- <AppenderRef ref="Console"/>
- </Logger>
- </Loggers>
-</Configuration>
\ No newline at end of file
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/tika-config-simple-emitter.xml b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/tika-config-simple-emitter.xml
deleted file mode 100644
index 5b14a5441..000000000
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/resources/tika-config-simple-emitter.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<properties>
- <metadataFilters>
- <metadataFilter class="org.apache.tika.metadata.filter.FieldNameMappingFilter">
- <mappings>
- <mapping from="X-TIKA:content" to="content"/>
- <mapping from="X-TIKA:embedded_resource_path" to="embedded_path"/>
- <mapping from="Content-Length" to="length"/>
- <mapping from="dc:creator" to="creators"/>
- <mapping from="dc:title" to="title"/>
- </mappings>
- </metadataFilter>
- </metadataFilters>
- <emitters>
- <emitter class="org.apache.tika.pipes.emitter.solr.SolrEmitter">
- <name>solr1</name>
- <url>http://localhost:8983/solr/tika-test</url>
- <attachmentStrategy>concatenate-content</attachmentStrategy>
- <contentField>content</contentField>
- <commitWithin>10</commitWithin>
- </emitter>
- <emitter class="org.apache.tika.pipes.emitter.solr.SolrEmitter">
- <name>solr2</name>
- <url>http://localhost:8983/solr/tika-test</url>
- <attachmentStrategy>parent-child</attachmentStrategy>
- <contentField>content</contentField>
- <commitWithin>10</commitWithin>
- </emitter>
- </emitters>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/pom.xml b/tika-pipes/tika-fetchers/pom.xml
deleted file mode 100644
index 999d269fc..000000000
--- a/tika-pipes/tika-fetchers/pom.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-pipes</artifactId>
- <version>4.0.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-fetchers</artifactId>
- <name>Apache Tika fetchers</name>
- <url>https://tika.apache.org/</url>
- <packaging>pom</packaging>
-
- <modules>
- <module>tika-fetcher-http</module>
- <module>tika-fetcher-s3</module>
- <module>tika-fetcher-gcs</module>
- <module>tika-fetcher-az-blob</module>
- <module>tika-fetcher-microsoft-graph</module>
- </modules>
-
- <dependencies>
-
- </dependencies>
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/pom.xml
deleted file mode 100644
index 2fd9153e3..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/pom.xml
+++ /dev/null
@@ -1,119 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-fetchers</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-fetcher-az-blob</artifactId>
- <name>Apache Tika Azure Blob fetcher</name>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.azure</groupId>
- <artifactId>azure-storage-blob</artifactId>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-serialization</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.fetcher.azblob</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/AZBlobFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/AZBlobFetcher.java
deleted file mode 100644
index 62f082b7f..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/AZBlobFetcher.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.azblob;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Path;
-import java.util.Map;
-
-import com.azure.storage.blob.BlobClient;
-import com.azure.storage.blob.BlobClientBuilder;
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.BlobServiceClient;
-import com.azure.storage.blob.BlobServiceClientBuilder;
-import com.azure.storage.blob.models.BlobProperties;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TemporaryResources;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
-import org.apache.tika.pipes.fetcher.azblob.config.AZBlobFetcherConfig;
-import org.apache.tika.utils.StringUtils;
-
-/**
- * Fetches files from Azure blob storage.
- * <p>
- * There are two modes:
- * 1) If you are only using one endpoint and one sas token and one container,
- * configure those in the config file. In this case, your fetchKey will
- * be the path in the container to the blob.
- * 2) If you have different endpoints or sas tokens or containers across
- * your requests, your fetchKey will be the complete SAS url pointing to the blob.
- */
-public class AZBlobFetcher extends AbstractFetcher implements Initializable {
- public AZBlobFetcher() {
-
- }
- public AZBlobFetcher(AZBlobFetcherConfig azBlobFetcherConfig) {
- setContainer(azBlobFetcherConfig.getContainer());
- setEndpoint(azBlobFetcherConfig.getEndpoint());
- setSasToken(azBlobFetcherConfig.getSasToken());
- setSpoolToTemp(azBlobFetcherConfig.isSpoolToTemp());
- setExtractUserMetadata(azBlobFetcherConfig.isExtractUserMetadata());
- }
-
- private static final Logger LOGGER = LoggerFactory.getLogger(AZBlobFetcher.class);
- private static String PREFIX = "az-blob";
- private String sasToken;
- private String container;
- private String endpoint;
- private BlobClientFactory blobClientFactory;
- private boolean extractUserMetadata = true;
- private BlobServiceClient blobServiceClient;
- private BlobContainerClient blobContainerClient;
- private boolean spoolToTemp = true;
-
- @Override
- public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws TikaException, IOException {
-
- LOGGER.debug("about to fetch fetchkey={} from endpoint ({})", fetchKey, endpoint);
-
- try {
- BlobClient blobClient = blobClientFactory.getClient(fetchKey);
-
- if (extractUserMetadata) {
- BlobProperties properties = blobClient.getProperties();
- if (properties.getMetadata() != null) {
- for (Map.Entry<String, String> e : properties
- .getMetadata()
- .entrySet()) {
- metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
- }
- }
- }
- if (!spoolToTemp) {
- return TikaInputStream.get(blobClient.openInputStream());
- } else {
- long start = System.currentTimeMillis();
- TemporaryResources tmpResources = new TemporaryResources();
- Path tmp = tmpResources.createTempFile();
- blobClient.downloadToFile(tmp.toRealPath().toString());
- TikaInputStream tis = TikaInputStream.get(tmp, metadata, tmpResources);
- long elapsed = System.currentTimeMillis() - start;
- LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
- return tis;
- }
- } catch (Exception e) {
- throw new IOException("az-blob storage exception", e);
- }
- }
-
- @Field
- public void setSpoolToTemp(boolean spoolToTemp) {
- this.spoolToTemp = spoolToTemp;
- }
-
- @Field
- public void setSasToken(String sasToken) {
- this.sasToken = sasToken;
- }
-
- @Field
- public void setEndpoint(String endpoint) {
- this.endpoint = endpoint;
- }
-
- @Field
- public void setContainer(String container) {
- this.container = container;
- }
-
- /**
- * Whether or not to extract user metadata from the blob object
- *
- * @param extractUserMetadata
- */
- @Field
- public void setExtractUserMetadata(boolean extractUserMetadata) {
- this.extractUserMetadata = extractUserMetadata;
- }
-
-
- /**
- * This initializes the az blob container client
- *
- * @param params params to use for initialization
- * @throws TikaConfigException
- */
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- if (!StringUtils.isBlank(sasToken)) {
- LOGGER.debug("Setting up immutable endpoint, token and container");
- blobClientFactory = new SingleBlobContainerFactory(endpoint, sasToken, container);
- } else {
- LOGGER.debug("Setting up blobclientfactory to recieve the full sas url for the blob");
- blobClientFactory = new SASUrlFactory();
- }
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
- //if the user has set one of these, they need to have set all of them
- if (!StringUtils.isBlank(this.sasToken) || !StringUtils.isBlank(this.endpoint) || !StringUtils.isBlank(this.container)) {
- mustNotBeEmpty("sasToken", this.sasToken);
- mustNotBeEmpty("endpoint", this.endpoint);
- mustNotBeEmpty("container", this.container);
- }
- }
-
- private interface BlobClientFactory {
- BlobClient getClient(String fetchKey);
- }
-
- private static class SingleBlobContainerFactory implements BlobClientFactory {
- private final BlobContainerClient blobContainerClient;
-
- private SingleBlobContainerFactory(String endpoint, String sasToken, String container) {
- //TODO -- allow authentication via other methods
- BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
- .endpoint(endpoint)
- .sasToken(sasToken)
- .buildClient();
- blobContainerClient = blobServiceClient.getBlobContainerClient(container);
- }
-
- @Override
- public BlobClient getClient(String fetchKey) {
- return blobContainerClient.getBlobClient(fetchKey);
- }
- }
-
- private static class SASUrlFactory implements BlobClientFactory {
-
- @Override
- public BlobClient getClient(String fetchKey) {
- return new BlobClientBuilder()
- .connectionString(fetchKey)
- .buildClient();
- }
- }
-}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/config/AZBlobFetcherConfig.java b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/config/AZBlobFetcherConfig.java
deleted file mode 100644
index e29e62319..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/config/AZBlobFetcherConfig.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.azblob.config;
-
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
-
-public class AZBlobFetcherConfig extends AbstractConfig {
- private boolean spoolToTemp;
- private String sasToken;
- private String endpoint;
- private String container;
- private boolean extractUserMetadata;
-
- public boolean isSpoolToTemp() {
- return spoolToTemp;
- }
-
- public AZBlobFetcherConfig setSpoolToTemp(boolean spoolToTemp) {
- this.spoolToTemp = spoolToTemp;
- return this;
- }
-
- public String getSasToken() {
- return sasToken;
- }
-
- public AZBlobFetcherConfig setSasToken(String sasToken) {
- this.sasToken = sasToken;
- return this;
- }
-
- public String getEndpoint() {
- return endpoint;
- }
-
- public AZBlobFetcherConfig setEndpoint(String endpoint) {
- this.endpoint = endpoint;
- return this;
- }
-
- public String getContainer() {
- return container;
- }
-
- public AZBlobFetcherConfig setContainer(String container) {
- this.container = container;
- return this;
- }
-
- public boolean isExtractUserMetadata() {
- return extractUserMetadata;
- }
-
- public AZBlobFetcherConfig setExtractUserMetadata(boolean extractUserMetadata) {
- this.extractUserMetadata = extractUserMetadata;
- return this;
- }
-}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/java/org/apache/tika/pipes/fetcher/azblob/TestAZBlobFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/java/org/apache/tika/pipes/fetcher/azblob/TestAZBlobFetcher.java
deleted file mode 100644
index bbfc63c65..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/java/org/apache/tika/pipes/fetcher/azblob/TestAZBlobFetcher.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.azblob;
-
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Paths;
-import java.util.List;
-
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.TikaTest;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.Fetcher;
-import org.apache.tika.pipes.core.fetcher.FetcherManager;
-import org.apache.tika.serialization.JsonMetadataList;
-
-@Disabled("write actual unit tests")
-public class TestAZBlobFetcher extends TikaTest {
-
- private static final String FETCH_STRING = "something-or-other/test-out.json";
-
- @Test
- public void testConfig() throws Exception {
- FetcherManager fetcherManager = FetcherManager.load(Paths.get(this
- .getClass()
- .getResource("/tika-config-az-blob.xml")
- .toURI()));
- Fetcher fetcher = fetcherManager.getFetcher("az-blob");
- List<Metadata> metadataList = null;
- try (Reader reader = new BufferedReader(new InputStreamReader(fetcher.fetch(FETCH_STRING, new Metadata(), new ParseContext()), StandardCharsets.UTF_8))) {
- metadataList = JsonMetadataList.fromJson(reader);
- }
- debug(metadataList);
- }
-}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/resources/tika-config-az-blob.xml b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/resources/tika-config-az-blob.xml
deleted file mode 100644
index 2aa6ba9a5..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/resources/tika-config-az-blob.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<properties>
- <fetchers>
- <fetcher class="org.apache.tika.pipes.fetcher.azblob.AZBlobFetcher">
- <name>az-blob</name>
- <!-- Either configure these three and send in the path for the blob OR
- do not configure any of these and send in the full SAS url for the blob
- as the fetchkey-->
- <endpoint></endpoint>
- <container></container>
- <sasToken></sasToken>
- </fetcher>
- </fetchers>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-gcs/pom.xml
deleted file mode 100644
index cbf4555e3..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/pom.xml
+++ /dev/null
@@ -1,106 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-fetchers</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-fetcher-gcs</artifactId>
- <name>Apache Tika Google Cloud Storage fetcher</name>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.cloud</groupId>
- <artifactId>google-cloud-storage</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.fetcher.gcs</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java
deleted file mode 100644
index 42e7cf486..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcher.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.gcs;
-
-import static org.apache.tika.config.TikaConfig.mustNotBeEmpty;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Path;
-import java.util.Map;
-
-import com.google.cloud.storage.Blob;
-import com.google.cloud.storage.BlobId;
-import com.google.cloud.storage.Storage;
-import com.google.cloud.storage.StorageOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TemporaryResources;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
-import org.apache.tika.pipes.fetcher.gcs.config.GCSFetcherConfig;
-
-/**
- * Fetches files from google cloud storage. Must set projectId and bucket via the config.
- */
-public class GCSFetcher extends AbstractFetcher implements Initializable {
- public GCSFetcher() {
-
- }
- public GCSFetcher(GCSFetcherConfig gcsFetcherConfig) {
- setBucket(gcsFetcherConfig.getBucket());
- setProjectId(gcsFetcherConfig.getProjectId());
- setSpoolToTemp(gcsFetcherConfig.isSpoolToTemp());
- setExtractUserMetadata(gcsFetcherConfig.isExtractUserMetadata());
- }
- private static String PREFIX = "gcs";
- private static final Logger LOGGER = LoggerFactory.getLogger(GCSFetcher.class);
- private String projectId;
- private String bucket;
- private boolean extractUserMetadata = true;
- private Storage storage;
- private boolean spoolToTemp = true;
-
- @Override
- public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws TikaException, IOException {
-
- LOGGER.debug("about to fetch fetchkey={} from bucket ({})", fetchKey, bucket);
-
- try {
- Blob blob = storage.get(BlobId.of(bucket, fetchKey));
-
- if (extractUserMetadata) {
- if (blob.getMetadata() != null) {
- for (Map.Entry<String, String> e : blob.getMetadata().entrySet()) {
- metadata.add(PREFIX + ":" + e.getKey(), e.getValue());
- }
- }
- }
- if (!spoolToTemp) {
- return TikaInputStream.get(blob.getContent());
- } else {
- long start = System.currentTimeMillis();
- TemporaryResources tmpResources = new TemporaryResources();
- Path tmp = tmpResources.createTempFile();
- blob.downloadTo(tmp);
- TikaInputStream tis = TikaInputStream.get(tmp, metadata, tmpResources);
- long elapsed = System.currentTimeMillis() - start;
- LOGGER.debug("took {} ms to copy to local tmp file", elapsed);
- return tis;
- }
- } catch (Exception e) {
- throw new IOException("gcs storage exception", e);
- }
- }
-
- @Field
- public void setSpoolToTemp(boolean spoolToTemp) {
- this.spoolToTemp = spoolToTemp;
- }
-
- @Field
- public void setProjectId(String projectId) {
- this.projectId = projectId;
- }
-
- @Field
- public void setBucket(String bucket) {
- this.bucket = bucket;
- }
-
- /**
- * Whether or not to extract user metadata from the S3Object
- *
- * @param extractUserMetadata
- */
- @Field
- public void setExtractUserMetadata(boolean extractUserMetadata) {
- this.extractUserMetadata = extractUserMetadata;
- }
-
- //TODO: parameterize extracting other blob metadata, eg. md5, crc, etc.
-
- /**
- * This initializes the gcs storage client.
- *
- * @param params params to use for initialization
- * @throws TikaConfigException
- */
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
- //params have already been set...ignore them
- //TODO -- add other params to the builder as needed
- storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler)
- throws TikaConfigException {
- mustNotBeEmpty("bucket", this.bucket);
- mustNotBeEmpty("projectId", this.projectId);
- }
-}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/config/GCSFetcherConfig.java b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/config/GCSFetcherConfig.java
deleted file mode 100644
index 6a808f1ec..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/config/GCSFetcherConfig.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.gcs.config;
-
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
-
-public class GCSFetcherConfig extends AbstractConfig {
- private boolean spoolToTemp;
- private String projectId;
- private String bucket;
- private boolean extractUserMetadata;
-
- public boolean isSpoolToTemp() {
- return spoolToTemp;
- }
-
- public GCSFetcherConfig setSpoolToTemp(boolean spoolToTemp) {
- this.spoolToTemp = spoolToTemp;
- return this;
- }
-
- public String getProjectId() {
- return projectId;
- }
-
- public GCSFetcherConfig setProjectId(String projectId) {
- this.projectId = projectId;
- return this;
- }
-
- public String getBucket() {
- return bucket;
- }
-
- public GCSFetcherConfig setBucket(String bucket) {
- this.bucket = bucket;
- return this;
- }
-
- public boolean isExtractUserMetadata() {
- return extractUserMetadata;
- }
-
- public GCSFetcherConfig setExtractUserMetadata(boolean extractUserMetadata) {
- this.extractUserMetadata = extractUserMetadata;
- return this;
- }
-}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/java/org/apache/tika/pipes/fetcher/s3/TestGCSFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/java/org/apache/tika/pipes/fetcher/s3/TestGCSFetcher.java
deleted file mode 100644
index 3844337f3..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/java/org/apache/tika/pipes/fetcher/s3/TestGCSFetcher.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.s3;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
-
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.Fetcher;
-import org.apache.tika.pipes.core.fetcher.FetcherManager;
-
-@Disabled("write actual unit tests")
-public class TestGCSFetcher {
-
- private static final String FETCH_STRING = "testExtraSpaces.pdf";
-
- @TempDir
- private static Path TEMP_DIR;
- private static Path outputFile;
-
- @BeforeAll
- public static void setUp() throws Exception {
- outputFile = Files.createTempFile(TEMP_DIR, "tika-test", ".pdf");
- }
-
-
- @Test
- public void testConfig() throws Exception {
- FetcherManager fetcherManager = FetcherManager.load(
- Paths.get(this.getClass().getResource("/tika-config-gcs.xml").toURI()));
- Fetcher fetcher = fetcherManager.getFetcher("gcs");
- Metadata metadata = new Metadata();
- try (InputStream is = fetcher.fetch(FETCH_STRING, metadata, new ParseContext())) {
- Files.copy(is, outputFile, StandardCopyOption.REPLACE_EXISTING);
- }
- assertEquals(20743, Files.size(outputFile));
- }
-}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/resources/tika-config-gcs.xml b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/resources/tika-config-gcs.xml
deleted file mode 100644
index 7e9ba6817..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/resources/tika-config-gcs.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<properties>
- <fetchers>
- <fetcher class="org.apache.tika.pipes.fetcher.gcs.GCSFetcher">
- <name>gcs</name>
- <projectId>My First Project</projectId>
- <bucket>tika-tallison-test-bucket</bucket>
- </fetcher>
- </fetchers>
-</properties>
\ No newline at end of file
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
deleted file mode 100644
index 00c27fc18..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
+++ /dev/null
@@ -1,139 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>tika-fetchers</artifactId>
- <groupId>org.apache.tika</groupId>
- <version>4.0.0-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>tika-fetcher-http</artifactId>
- <name>Apache Tika http fetcher</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-pipes-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-httpclient-commons</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.nimbusds</groupId>
- <artifactId>nimbus-jose-jwt</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Automatic-Module-Name>org.apache.tika.pipes.fetcher.http</Automatic-Module-Name>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.version}</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>
- false
- </createDependencyReducedPom>
- <!-- <filters> -->
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/LICENSE</resource>
- <file>target/classes/META-INF/LICENSE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/NOTICE</resource>
- <file>target/classes/META-INF/NOTICE</file>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
- <resource>META-INF/DEPENDENCIES</resource>
- <file>target/classes/META-INF/DEPENDENCIES</file>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <scm>
- <tag>3.0.0-rc1</tag>
- </scm>
-</project>
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
deleted file mode 100644
index ee1953cb7..000000000
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
+++ /dev/null
@@ -1,690 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.fetcher.http;
-
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.InetAddress;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.security.PrivateKey;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.nimbusds.jose.JOSEException;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
-import org.apache.http.ConnectionClosedException;
-import org.apache.http.Header;
-import org.apache.http.HttpConnection;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpInetConnection;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.impl.conn.ConnectionShutdownException;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.tika.client.HttpClientFactory;
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.exception.TikaTimeoutException;
-import org.apache.tika.io.TemporaryResources;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.Property;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
-import org.apache.tika.pipes.core.fetcher.RangeFetcher;
-import org.apache.tika.pipes.core.fetcher.config.FetcherConfigContainer;
-import org.apache.tika.pipes.fetcher.http.config.HttpFetcherConfig;
-import org.apache.tika.pipes.fetcher.http.config.HttpHeaders;
-import org.apache.tika.pipes.fetcher.http.jwt.JwtGenerator;
-import org.apache.tika.pipes.fetcher.http.jwt.JwtPrivateKeyCreds;
-import org.apache.tika.pipes.fetcher.http.jwt.JwtSecretCreds;
-import org.apache.tika.utils.StringUtils;
-
-/**
- * Based on Apache httpclient
- */
-public class HttpFetcher extends AbstractFetcher implements Initializable, RangeFetcher {
- public HttpFetcher() {
-
- }
-
- private static final ObjectMapper OM = new ObjectMapper();
- private HttpFetcherConfig httpFetcherConfig = new HttpFetcherConfig();
- private HttpClientFactory httpClientFactory = new HttpClientFactory();
-
- public HttpFetcher(HttpFetcherConfig httpFetcherConfig) {
- this.httpFetcherConfig = httpFetcherConfig;
- }
-
- public static String HTTP_HEADER_PREFIX = "http-header:";
-
- public static String HTTP_FETCH_PREFIX = "http-connection:";
-
- /**
- * http status code
- */
- public static Property HTTP_STATUS_CODE = Property.externalInteger(HTTP_HEADER_PREFIX + "status-code");
- /**
- * Number of redirects
- */
- public static Property HTTP_NUM_REDIRECTS = Property.externalInteger(HTTP_FETCH_PREFIX + "num-redirects");
-
- /**
- * If there were redirects, this captures the final URL visited
- */
- public static Property HTTP_TARGET_URL = Property.externalText(HTTP_FETCH_PREFIX + "target-url");
-
- public static Property HTTP_TARGET_IP_ADDRESS = Property.externalText(HTTP_FETCH_PREFIX + "target-ip-address");
-
- public static Property HTTP_FETCH_TRUNCATED = Property.externalBoolean(HTTP_FETCH_PREFIX + "fetch-truncated");
-
- public static Property HTTP_CONTENT_ENCODING = Property.externalText(HTTP_HEADER_PREFIX + "content-encoding");
-
- public static Property HTTP_CONTENT_TYPE = Property.externalText(HTTP_HEADER_PREFIX + "content-type");
-
- private static String USER_AGENT = "User-Agent";
-
- Logger LOG = LoggerFactory.getLogger(HttpFetcher.class);
-
- private HttpClient httpClient;
- //back-off client that disables compression
- private HttpClient noCompressHttpClient;
- private int maxRedirects = 10;
- //overall timeout in milliseconds
- private long overallTimeout = -1;
-
- private long maxSpoolSize = -1;
-
- //max string length to read from a result if the
- //status code was not in the 200 range
- private int maxErrMsgSize = 10000;
-
- //httpHeaders to capture in the metadata
- private Set<String> httpHeaders = new HashSet<>();
-
- //httpRequestHeaders to add to all outgoing http requests
- private Set<String> httpRequestHeaders = new HashSet<>();
-
- //When making the request, what User-Agent is sent.
- //By default httpclient adds e.g. "Apache-HttpClient/4.5.13 (Java/x.y.z)"
- private String userAgent = null;
-
- JwtGenerator jwtGenerator;
-
- @Override
- public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws IOException, TikaException {
- HttpFetcherConfig additionalHttpFetcherConfig = getAdditionalHttpFetcherConfig(parseContext);
- HttpGet get = new HttpGet(fetchKey);
- RequestConfig requestConfig = RequestConfig
- .custom()
- .setMaxRedirects(httpFetcherConfig.getMaxRedirects())
- .setRedirectsEnabled(httpFetcherConfig.getMaxRedirects() > 0)
- .build();
- get.setConfig(requestConfig);
- setHttpRequestHeaders(metadata, get);
- putAdditionalHeadersOnRequest(additionalHttpFetcherConfig, get);
- return execute(get, metadata, httpClient, true);
- }
-
- private void setHttpRequestHeaders(Metadata metadata, HttpGet get) {
- if (!StringUtils.isBlank(userAgent)) {
- get.setHeader(USER_AGENT, userAgent);
- }
- // Add the headers from the Fetcher configuration.
- if (httpRequestHeaders != null) {
- for (String httpRequestHeader : httpRequestHeaders) {
- parseHeaderAndPutOnRequest(get, httpRequestHeader);
- }
- }
- // Additionally, headers can be specified per-fetch via the metadata.
- String[] httpRequestHeaders = metadata.getValues("httpRequestHeaders");
- if (httpRequestHeaders != null) {
- for (String httpRequestHeader : httpRequestHeaders) {
- parseHeaderAndPutOnRequest(get, httpRequestHeader);
- }
- }
- }
-
- private static void parseHeaderAndPutOnRequest(HttpGet get, String httpRequestHeader) {
- String[] parts = httpRequestHeader
- .trim().split(":", 2);
- if (parts.length >= 2) {
- String key = parts[0].trim();
- String value = parts[1].trim();
- get.setHeader(key, value);
- }
- }
-
- private HttpFetcherConfig getAdditionalHttpFetcherConfig(ParseContext parseContext) throws JsonProcessingException {
- HttpFetcherConfig additionalHttpFetcherConfig = null;
- FetcherConfigContainer fetcherConfigContainer = parseContext.get(FetcherConfigContainer.class);
- if (fetcherConfigContainer != null) {
- additionalHttpFetcherConfig = OM.readValue(fetcherConfigContainer.getJson(), HttpFetcherConfig.class);
- }
- return additionalHttpFetcherConfig;
- }
-
- @Override
- public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata,
- ParseContext parseContext) throws IOException, TikaException {
- HttpFetcherConfig additionalHttpFetcherConfig = getAdditionalHttpFetcherConfig(parseContext);
- HttpGet get = new HttpGet(fetchKey);
- putAdditionalHeadersOnRequest(additionalHttpFetcherConfig, get);
-
- get.setHeader("Range", "bytes=" + startRange + "-" + endRange);
- return execute(get, metadata, httpClient, true);
- }
-
- private void putAdditionalHeadersOnRequest(HttpFetcherConfig additionalFetcherConfig, HttpGet httpGet) throws TikaException {
- if (!StringUtils.isBlank(httpFetcherConfig.getUserAgent())) {
- httpGet.setHeader(USER_AGENT, httpFetcherConfig.getUserAgent());
- }
- if (additionalFetcherConfig != null && additionalFetcherConfig.getHttpRequestHeaders() != null) {
- additionalFetcherConfig.getHttpRequestHeaders()
- .getHeaders()
- .forEach(httpGet::addHeader);
- }
- if (httpFetcherConfig.getHttpRequestHeaders() != null) {
- httpFetcherConfig.getHttpRequestHeaders()
- .getHeaders()
- .forEach(httpGet::addHeader);
- }
- if (jwtGenerator != null) {
- try {
- httpGet.setHeader("Authorization", "Bearer " + jwtGenerator.jwt());
- } catch (JOSEException e) {
... 37346 lines suppressed ...