You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by nd...@apache.org on 2024/06/23 19:55:23 UTC
(tika) branch tika-grpc-3x-features updated (e50944daf -> 4fb194e66)
This is an automated email from the ASF dual-hosted git repository.
ndipiazza pushed a change to branch tika-grpc-3x-features
in repository https://gitbox.apache.org/repos/asf/tika.git
from e50944daf Merge branch 'TIKA-4181-grpc' into tika-grpc-3x-features
add b808f9098 TIKA-4252: add protection against null metadata
add de282d286 TIKA-4248 -- improve handling of attachments in PST (#1738)
add d501c0197 Avoid race condition in NamedEntityParserTest
add 9f8a2f58b TIKA-4249 -- allow utf8 bom to at start of rfc822 detection (#1739)
add 5d8b2e0aa Bump org.apache.maven.plugin-tools:maven-plugin-annotations
add fd60b7905 Merge pull request #1749 from apache/dependabot/maven/org.apache.maven.plugin-tools-maven-plugin-annotations-3.13.0
add 643307da2 Bump com.azure:azure-core from 1.48.0 to 1.49.0
add 6e1354275 Merge pull request #1748 from apache/dependabot/maven/com.azure-azure-core-1.49.0
add 9086f937d Bump aws.version from 1.12.710 to 1.12.715
add 08a512443 Merge pull request #1747 from apache/dependabot/maven/aws.version-1.12.715
add 4931f3f3a Bump org.apache.commons:commons-csv from 1.10.0 to 1.11.0
add 224fdd014 Merge pull request #1744 from apache/dependabot/maven/org.apache.commons-commons-csv-1.11.0
add 5b77317d2 Bump com.fasterxml.jackson:jackson-bom from 2.17.0 to 2.17.1
add 606045715 Merge pull request #1741 from apache/dependabot/maven/com.fasterxml.jackson-jackson-bom-2.17.1
add 27c40c8c0 Bump com.google.guava:guava from 33.1.0-jre to 33.2.0-jre
add 5f3aac2cf Merge pull request #1740 from apache/dependabot/maven/com.google.guava-guava-33.2.0-jre
add 28dce0b75 Bump org.scala-lang:scala-reflect from 2.13.13 to 2.13.14
add 9b9032ad3 Merge pull request #1745 from apache/dependabot/maven/org.scala-lang-scala-reflect-2.13.14
add df81bd442 Bump com.azure:azure-core-http-netty from 1.14.2 to 1.15.0
add 7752c921f Merge pull request #1743 from apache/dependabot/maven/com.azure-azure-core-http-netty-1.15.0
add 4970d12e7 Bump com.mchange:mchange-commons-java from 0.3.0 to 0.3.1
add e172306ef Merge pull request #1742 from apache/dependabot/maven/com.mchange-mchange-commons-java-0.3.1
add 48c934f54 Fix small logic bug in JDBCEmitter in calculating when to truncate.
add ebdb6fc87 TIKA-4220 -- temporary workaround for tar detection regression (#1687) -- main branch
add 84d05ad4a TIKA-4221 -- temporary workaround for COMPRESS-675 (#1686) -- fix merge conflicts (#1750)
add 2f8dbdfbd TIKA-4252: fix metadata issue (#1752)
add 32baf2345 TIKA-4250 -- add optional parser for pst files -- wrapper for libpst/readpst (#1751)
add b068e4290 TIKA-4252: add request metadata (#1753)
add 3afb7eeee Revert bad commits (#1755)
add 1b53c5706 Bump aws.version from 1.12.715 to 1.12.720
add 0635ae0d3 Merge pull request #1760 from apache/dependabot/maven/aws.version-1.12.720
add 04209e54c Bump com.mchange:c3p0 from 0.9.5.5 to 0.10.1
add b99103a38 Merge pull request #1759 from apache/dependabot/maven/com.mchange-c3p0-0.10.1
add 5926c69cf Bump org.testcontainers:testcontainers-bom from 1.19.7 to 1.19.8
add fe2050e0d Merge pull request #1758 from apache/dependabot/maven/org.testcontainers-testcontainers-bom-1.19.8
add be73f4da3 Bump com.healthmarketscience.jackcess:jackcess from 4.0.5 to 4.0.6
add abe8d7321 Merge pull request #1757 from apache/dependabot/maven/com.healthmarketscience.jackcess-jackcess-4.0.6
add fe6861428 Bump com.google.cloud:google-cloud-storage from 2.37.0 to 2.38.0
add dcd260664 Merge pull request #1756 from apache/dependabot/maven/com.google.cloud-google-cloud-storage-2.38.0
add 103427279 Bump aws.version from 1.12.720 to 1.12.725
add 05109e4c8 Merge pull request #1763 from apache/dependabot/maven/aws.version-1.12.725
add 302092a05 Bump org.springframework:spring-context from 5.3.34 to 5.3.35
add 7011788c7 Merge pull request #1766 from apache/dependabot/maven/org.springframework-spring-context-5.3.35
add 9f8fad200 Bump reactor.netty.version from 1.1.18 to 1.1.19
add f523d7069 Merge pull request #1769 from apache/dependabot/maven/reactor.netty.version-1.1.19
add 1273a02ec Bump jakarta.xml.soap:jakarta.xml.soap-api from 3.0.1 to 3.0.2
add 95a7012cd Merge pull request #1772 from apache/dependabot/maven/jakarta.xml.soap-jakarta.xml.soap-api-3.0.2
add 9788fb977 Bump io.projectreactor:reactor-core from 3.6.5 to 3.6.6
add 4ce269952 Merge pull request #1767 from apache/dependabot/maven/io.projectreactor-reactor-core-3.6.6
add 02c34398e Bump commons-logging:commons-logging from 1.3.1 to 1.3.2
add 3f53a9de2 Merge pull request #1764 from apache/dependabot/maven/commons-logging-commons-logging-1.3.2
add ea0e08fa7 TIKA-4166: add version and exclusion for hdf5
add 41d3eafe5 TIKA-4166: add version of uimaj-cpe
add 22f9533cf TIKA-4166: add version of error_prone_annotations
add 08b8632fa Bump org.apache.ctakes:ctakes-core from 4.0.0.1 to 5.1.0
add 847c3ea5b Merge pull request #1765 from apache/dependabot/maven/org.apache.ctakes-ctakes-core-5.1.0
add caa32f92c Bump com.google.code.gson:gson from 2.10.1 to 2.11.0
add db0073369 Merge pull request #1768 from apache/dependabot/maven/com.google.code.gson-gson-2.11.0
add e4e28e3ba Bump com.azure:azure-storage-blob from 12.25.4 to 12.26.0
add cc2bb877d Merge pull request #1770 from apache/dependabot/maven/com.azure-azure-storage-blob-12.26.0
add 9cbdaa737 Bump org.codehaus.mojo:build-helper-maven-plugin from 3.5.0 to 3.6.0
add 8ae589e1d Merge pull request #1771 from apache/dependabot/maven/org.codehaus.mojo-build-helper-maven-plugin-3.6.0
add 15c7ec390 TIKA-4166: update junit, mockito, error_prone_annotations
add c6f2d14fa TIKA-4166: add comment, remove entry that is no longer needed
add 429fb9a00 TIKA-4166: update comment
add 7a03331f8 TIKA-4256 -- allow inlining of ocr'd content in the RecursiveParserWrapper (#1762)
add 019041117 TIKA-4257 -- lower dbf priority (#1773)
add 84dc15e7d TIKA-4166: add / move comment, update jetty, aws
add 035682cdd TIKA-4259 (#1775)
add c1f07222b TIKA-4261 -- add a clear by attachment type metadata filter (#1777)
add e03d3ecb1 Bump org.apache.maven:maven-model from 3.9.6 to 3.9.7
add b47b1cdc4 Merge pull request #1779 from apache/dependabot/maven/org.apache.maven-maven-model-3.9.7
add 423076d44 Bump org.springframework:spring-context from 5.3.35 to 5.3.36
add 55563d03e Merge pull request #1785 from apache/dependabot/maven/org.springframework-spring-context-5.3.36
add d1ef460ce Bump aws.version from 1.12.726 to 1.12.730
add 36e7649f6 Merge pull request #1787 from apache/dependabot/maven/aws.version-1.12.730
add 4fb3932d7 Bump com.google.cloud:google-cloud-storage from 2.38.0 to 2.39.0
add ba19bcb5d Merge pull request #1786 from apache/dependabot/maven/com.google.cloud-google-cloud-storage-2.39.0
add a7de46c63 Bump org.apache.commons:commons-compress from 1.26.1 to 1.26.2
add def412de1 Merge pull request #1784 from apache/dependabot/maven/org.apache.commons-commons-compress-1.26.2
add ad7780ac8 Bump jakarta.websocket:jakarta.websocket-api from 2.1.1 to 2.2.0
add e71c70666 Merge pull request #1783 from apache/dependabot/maven/jakarta.websocket-jakarta.websocket-api-2.2.0
add 2ce48fc89 Bump io.netty:netty-bom from 4.1.109.Final to 4.1.110.Final
add b52a598ef Merge pull request #1782 from apache/dependabot/maven/io.netty-netty-bom-4.1.110.Final
add 0f0804c26 Bump commons-cli:commons-cli from 1.7.0 to 1.8.0
add 13a1899da Merge pull request #1780 from apache/dependabot/maven/commons-cli-commons-cli-1.8.0
add b56c88b44 Bump org.codehaus.mojo:exec-maven-plugin from 3.2.0 to 3.3.0
add 27af36983 Merge pull request #1781 from apache/dependabot/maven/org.codehaus.mojo-exec-maven-plugin-3.3.0
add 095255866 TIKA-4221 -- revert workaround (#1789)
add c876d2932 TIKA-4220 -- revert workaround (#1790)
add 994f90ec7 Bump aws.version from 1.12.730 to 1.12.734
add 859376cec Merge pull request #1794 from apache/dependabot/maven/aws.version-1.12.734
add 4be65374b Bump org.apache.maven.plugins:maven-shade-plugin from 3.5.3 to 3.6.0
add 5208779bf Merge pull request #1803 from apache/dependabot/maven/org.apache.maven.plugins-maven-shade-plugin-3.6.0
add bc4bcdc76 Bump org.apache.maven.plugin-tools:maven-plugin-annotations
add f64bfe9b6 Merge pull request #1793 from apache/dependabot/maven/org.apache.maven.plugin-tools-maven-plugin-annotations-3.13.1
add fadfdf0ef Bump org.apache.maven.plugins:maven-enforcer-plugin from 3.4.1 to 3.5.0
add a965a158f Merge pull request #1792 from apache/dependabot/maven/org.apache.maven.plugins-maven-enforcer-plugin-3.5.0
add e05f93ddf Bump com.google.errorprone:error_prone_annotations from 2.27.1 to 2.28.0
add 1ebbf3cc2 Merge pull request #1801 from apache/dependabot/maven/com.google.errorprone-error_prone_annotations-2.28.0
add 81bf80c6c Bump org.apache.jackrabbit:oak-jackrabbit-api from 1.62.0 to 1.64.0
add ee7b2a6d9 Merge pull request #1802 from apache/dependabot/maven/org.apache.jackrabbit-oak-jackrabbit-api-1.64.0
add e634a3981 Bump org.freemarker:freemarker from 2.3.32 to 2.3.33
add 288738e22 Merge pull request #1800 from apache/dependabot/maven/org.freemarker-freemarker-2.3.33
add 1c04f3b4d Bump org.apache.maven.plugins:maven-javadoc-plugin from 3.6.3 to 3.7.0
add b52a76ec0 Merge pull request #1799 from apache/dependabot/maven/org.apache.maven.plugins-maven-javadoc-plugin-3.7.0
add 1316ab1f5 Bump commons-net:commons-net from 3.10.0 to 3.11.0
add a53855ccd Merge pull request #1798 from apache/dependabot/maven/commons-net-commons-net-3.11.0
add 310627a19 Bump com.google.guava:guava from 33.2.0-jre to 33.2.1-jre
add 4edbd4ee8 Merge pull request #1797 from apache/dependabot/maven/com.google.guava-guava-33.2.1-jre
add ee2e9f93f Bump org.xerial:sqlite-jdbc from 3.45.3.0 to 3.46.0.0
add 21078117b Merge pull request #1796 from apache/dependabot/maven/org.xerial-sqlite-jdbc-3.46.0.0
add c70bb4201 Bump com.github.albfernandez:juniversalchardet from 2.4.0 to 2.5.0
add b6a111290 Merge pull request #1795 from apache/dependabot/maven/com.github.albfernandez-juniversalchardet-2.5.0
add 6f626d252 TIKA-4252: switch to using the parse context for additional http headers (#1778)
add dd3e7ee40 Tika-4268 -- improve embedded resource path naming, esp for msg and email-based formats (#1804)
add c3f637a0c Bump com.azure:azure-core from 1.49.0 to 1.49.1
add eddc42b27 Merge pull request #1812 from apache/dependabot/maven/com.azure-azure-core-1.49.1
add 5916be47d Bump com.google.cloud:google-cloud-storage from 2.39.0 to 2.40.0
add 8150b1020 Merge pull request #1810 from apache/dependabot/maven/com.google.cloud-google-cloud-storage-2.40.0
add 06995cd47 Bump com.azure:azure-storage-blob from 12.26.0 to 12.26.1
add 2e8b72750 Merge pull request #1811 from apache/dependabot/maven/com.azure-azure-storage-blob-12.26.1
add 2f5d11372 Bump org.apache.maven.plugins:maven-checkstyle-plugin
add ce070aea0 Merge pull request #1809 from apache/dependabot/maven/org.apache.maven.plugins-maven-checkstyle-plugin-3.4.0
add eb3470398 Bump lucene.version from 9.10.0 to 9.11.0
add cf65c2621 Merge pull request #1807 from apache/dependabot/maven/lucene.version-9.11.0
add b4bb62e92 Bump twelvemonkeys.version from 3.10.1 to 3.11.0
add 1efe51078 Merge pull request #1806 from apache/dependabot/maven/twelvemonkeys.version-3.11.0
add 01c25954f Bump aws.version from 1.12.734 to 1.12.739
add e64584b2b Merge pull request #1808 from apache/dependabot/maven/aws.version-1.12.739
add 42158ce9d Bump com.azure:azure-core-http-netty from 1.15.0 to 1.15.1
add 6a493cb59 Merge pull request #1813 from apache/dependabot/maven/com.azure-azure-core-http-netty-1.15.1
add f3d4095bf Bump org.springframework:spring-context from 5.3.36 to 5.3.37
add 5a597411b Merge pull request #1823 from apache/dependabot/maven/org.springframework-spring-context-5.3.37
add 5be61f175 Bump aws.version from 1.12.739 to 1.12.744
add afeb6394f Merge pull request #1819 from apache/dependabot/maven/aws.version-1.12.744
add d8a692675 Bump org.apache.maven.plugins:maven-failsafe-plugin from 3.2.5 to 3.3.0
add db5a2926e Merge pull request #1821 from apache/dependabot/maven/org.apache.maven.plugins-maven-failsafe-plugin-3.3.0
add b857f0b01 Bump org.apache.maven.plugins:maven-surefire-plugin from 3.2.5 to 3.3.0
add 674e8ecbe Merge pull request #1822 from apache/dependabot/maven/org.apache.maven.plugins-maven-surefire-plugin-3.3.0
add 32647600a Bump io.projectreactor:reactor-core from 3.6.6 to 3.6.7
add beee07013 Merge pull request #1825 from apache/dependabot/maven/io.projectreactor-reactor-core-3.6.7
add aca6a6114 Bump org.apache.curator:curator-recipes from 5.6.0 to 5.7.0
add 1a7571e6d Merge pull request #1824 from apache/dependabot/maven/org.apache.curator-curator-recipes-5.7.0
add be25e2bc2 Bump io.netty:netty-bom from 4.1.110.Final to 4.1.111.Final
add f2c90c9f6 Merge pull request #1820 from apache/dependabot/maven/io.netty-netty-bom-4.1.111.Final
add d7b8e3c86 Bump jackrabbit.version from 2.21.25 to 2.22.0
add cbba8c5a7 Merge pull request #1818 from apache/dependabot/maven/jackrabbit.version-2.22.0
add 14d45cb02 Bump org.apache.maven.plugins:maven-release-plugin from 3.0.1 to 3.1.0
add a0b2bd456 Merge pull request #1817 from apache/dependabot/maven/org.apache.maven.plugins-maven-release-plugin-3.1.0
add 77d0fd451 Bump commons-net:commons-net from 3.11.0 to 3.11.1
add e286148b9 Merge pull request #1816 from apache/dependabot/maven/commons-net-commons-net-3.11.1
add aa23a68a2 Bump org.apache.maven.plugins:maven-project-info-reports-plugin
add 4ca1ddb6f Merge pull request #1815 from apache/dependabot/maven/org.apache.maven.plugins-maven-project-info-reports-plugin-3.6.0
add 5bf5dc85e Bump reactor.netty.version from 1.1.19 to 1.1.20
add 3de91caca Merge pull request #1814 from apache/dependabot/maven/reactor.netty.version-1.1.20
add 14e22593d Merge branch 'main' of github.com:apache/tika into TIKA-4181-grpc
add 23d9bd1b6 TIKA-4252: fix merge conflicts from main
add ba92f9dd3 TIKA-4252: add json schema methods
add e4ababe9c TIKA-4252: fix broken tests, useless method
add f2f876b37 TIKA-4252: remove stupid sleep from test
add 0932e7bfe TIKA-4252: fix checkstyle issue
add 58c6933ae TIKA-4252: log violations to console
add c1f9f29dd TIKA-4252: log violations to console
add 878f767c6 TIKA-4252: exclude generated sources
add dfc9edbb0 TIKA-4252: if tika config is read-only, use a tmp file for tika server so that it can be modified
add c520d0244 TIKA-4252: if tika config is read-only, use a tmp file for tika server so that it can be modified
new 63583e025 Merge branch 'TIKA-4181-grpc' into tika-grpc-3x-features
new 4fb194e66 resolve conflicts
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.txt | 14 +
tika-app/pom.xml | 2 +-
.../batch/DigestingAutoDetectParserFactory.java | 9 +-
.../batch/builders/AppParserFactoryBuilder.java | 17 +-
.../apache/tika/cli/BatchCommandLineBuilder.java | 25 +-
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 258 +-
.../apache/tika/gui/ParsingTransferHandler.java | 9 +-
.../src/main/java/org/apache/tika/gui/TikaGUI.java | 78 +-
.../java/org/apache/tika/cli/TikaCLIAsyncTest.java | 36 +-
.../tika/cli/TikaCLIBatchCommandLineTest.java | 24 +-
.../tika/cli/TikaCLIBatchIntegrationTest.java | 65 +-
.../test/java/org/apache/tika/cli/TikaCLITest.java | 126 +-
.../tika/extractor/TestEmbeddedDocumentUtil.java | 8 +-
.../java/org/apache/tika/batch/BatchProcess.java | 111 +-
.../apache/tika/batch/BatchProcessDriverCLI.java | 44 +-
.../apache/tika/batch/FileResourceConsumer.java | 52 +-
.../org/apache/tika/batch/FileResourceCrawler.java | 20 +-
.../tika/batch/ParallelFileProcessingResult.java | 10 +-
.../java/org/apache/tika/batch/StatusReporter.java | 25 +-
.../batch/builders/AbstractConsumersBuilder.java | 7 +-
.../tika/batch/builders/BatchProcessBuilder.java | 64 +-
.../batch/builders/CommandLineParserBuilder.java | 20 +-
.../DefaultContentHandlerFactoryBuilder.java | 3 +-
.../tika/batch/builders/ICrawlerBuilder.java | 3 +-
.../tika/batch/builders/InterrupterBuilder.java | 3 +-
.../builders/ObjectFromDOMAndQueueBuilder.java | 3 +-
.../tika/batch/builders/ParserFactoryBuilder.java | 7 +-
.../batch/builders/SimpleLogReporterBuilder.java | 6 +-
.../tika/batch/builders/StatusReporterBuilder.java | 3 +-
.../apache/tika/batch/fs/AbstractFSConsumer.java | 7 +-
.../apache/tika/batch/fs/BasicTikaFSConsumer.java | 16 +-
.../apache/tika/batch/fs/FSBatchProcessCLI.java | 15 +-
.../apache/tika/batch/fs/FSDirectoryCrawler.java | 30 +-
.../apache/tika/batch/fs/FSDocumentSelector.java | 3 +-
.../org/apache/tika/batch/fs/FSFileResource.java | 20 +-
.../org/apache/tika/batch/fs/FSListCrawler.java | 7 +-
.../tika/batch/fs/FSOutputStreamFactory.java | 9 +-
.../org/apache/tika/batch/fs/FSProperties.java | 3 +-
.../main/java/org/apache/tika/batch/fs/FSUtil.java | 36 +-
.../batch/fs/RecursiveParserWrapperFSConsumer.java | 13 +-
.../tika/batch/fs/StreamOutRPWFSConsumer.java | 26 +-
.../fs/builders/BasicTikaFSConsumersBuilder.java | 73 +-
.../tika/batch/fs/builders/FSCrawlerBuilder.java | 39 +-
.../batch/fs/strawman/StrawManTikaAppDriver.java | 46 +-
.../java/org/apache/tika/util/ClassLoaderUtil.java | 10 +-
.../main/java/org/apache/tika/util/PropsUtil.java | 8 +-
.../main/java/org/apache/tika/util/XMLDOMUtil.java | 19 +-
.../RecursiveParserWrapperFSConsumerTest.java | 82 +-
.../org/apache/tika/batch/fs/BatchDriverTest.java | 25 +-
.../org/apache/tika/batch/fs/BatchProcessTest.java | 126 +-
.../org/apache/tika/batch/fs/FSBatchTestBase.java | 37 +-
.../apache/tika/batch/fs/FSFileResourceTest.java | 10 +-
.../tika/batch/mock/MockConsumersBuilder.java | 3 +-
.../ParentContentHandler.java} | 22 +-
.../main/java/org/apache/tika/metadata/Office.java | 8 +
.../tika/metadata/{Rendering.java => PST.java} | 10 +-
.../apache/tika/metadata/TikaCoreProperties.java | 32 +-
.../ClearByAttachmentTypeMetadataFilter.java | 85 +
.../metadata/filter/CompositeMetadataFilter.java | 5 +
.../metadata/filter/FieldNameMappingFilter.java | 5 +
.../java/org/apache/tika/parser/ParseContext.java | 177 +-
.../apache/tika/parser/RecursiveParserWrapper.java | 102 +-
.../java/org/apache/tika/pipes/FetchEmitTuple.java | 91 +-
.../java/org/apache/tika/pipes/HandlerConfig.java | 63 +-
.../java/org/apache/tika/pipes/PipesClient.java | 5 +-
.../java/org/apache/tika/pipes/PipesServer.java | 58 +-
.../org/apache/tika/pipes/async/AsyncEmitter.java | 2 +-
.../apache/tika/pipes/emitter/AbstractEmitter.java | 6 +-
.../org/apache/tika/pipes/emitter/EmitData.java | 16 +-
.../org/apache/tika/pipes/emitter/EmitKey.java | 2 +
.../org/apache/tika/pipes/emitter/Emitter.java | 3 +-
.../apache/tika/pipes/emitter/EmptyEmitter.java | 3 +-
.../apache/tika/pipes/emitter/StreamEmitter.java | 3 +-
.../extractor/EmbeddedDocumentBytesConfig.java | 5 +-
.../EmittingEmbeddedDocumentBytesHandler.java | 17 +-
.../apache/tika/pipes/fetcher/EmptyFetcher.java | 3 +-
.../org/apache/tika/pipes/fetcher/Fetcher.java | 3 +-
.../apache/tika/pipes/fetcher/RangeFetcher.java | 8 +-
.../tika/pipes/fetcher/fs/FileSystemFetcher.java | 6 +-
.../apache/tika/pipes/fetcher/url/UrlFetcher.java | 3 +-
.../filelist/FileListPipesIterator.java | 6 +-
.../pipesiterator/fs/FileSystemPipesIterator.java | 6 +-
.../tika/sax/RecursiveParserWrapperHandler.java | 37 +-
.../java/org/apache/tika/utils/XMLReaderUtils.java | 59 +
.../org/apache/tika/mime/tika-mimetypes.xml | 25 +-
.../tika/metadata/filter/TestMetadataFilter.java | 19 +
.../org/apache/tika/mime/MimeDetectionTest.java | 23 +
.../org/apache/tika/parser/mock/MockParser.java | 3 +-
.../org/apache/tika/pipes/PipesClientTest.java | 3 +-
.../org/apache/tika/pipes/PipesServerTest.java | 18 +-
.../org/apache/tika/pipes/async/MockEmitter.java | 5 +-
.../org/apache/tika/pipes/async/MockFetcher.java | 3 +-
.../org/apache/tika/pipes/emitter/MockEmitter.java | 3 +-
.../org/apache/tika/pipes/fetcher/MockFetcher.java | 3 +-
...de.xml => TIKA-4261-clear-by-embedded-type.xml} | 10 +-
.../org/apache/tika/eval/app/AbstractProfiler.java | 180 +-
.../org/apache/tika/eval/app/EvalFilePaths.java | 3 +-
.../org/apache/tika/eval/app/ExtractComparer.java | 187 +-
.../org/apache/tika/eval/app/ExtractProfiler.java | 152 +-
.../org/apache/tika/eval/app/FileProfiler.java | 48 +-
.../java/org/apache/tika/eval/app/TikaEvalCLI.java | 70 +-
.../apache/tika/eval/app/XMLErrorLogUpdater.java | 43 +-
.../tika/eval/app/batch/DBConsumersManager.java | 3 +-
.../tika/eval/app/batch/EvalConsumerBuilder.java | 23 +-
.../tika/eval/app/batch/EvalConsumersBuilder.java | 6 +-
.../eval/app/batch/ExtractComparerBuilder.java | 9 +-
.../eval/app/batch/ExtractProfilerBuilder.java | 7 +-
.../java/org/apache/tika/eval/app/db/Cols.java | 25 +-
.../java/org/apache/tika/eval/app/db/DBBuffer.java | 7 +-
.../java/org/apache/tika/eval/app/db/H2Util.java | 4 +-
.../java/org/apache/tika/eval/app/db/JDBCUtil.java | 29 +-
.../org/apache/tika/eval/app/db/MimeBuffer.java | 10 +-
.../java/org/apache/tika/eval/app/io/DBWriter.java | 13 +-
.../org/apache/tika/eval/app/io/ExtractReader.java | 40 +-
.../tika/eval/app/io/ExtractReaderException.java | 3 +-
.../org/apache/tika/eval/app/io/XMLLogReader.java | 3 +-
.../org/apache/tika/eval/app/reports/Report.java | 9 +-
.../tika/eval/app/reports/ResultsReporter.java | 80 +-
.../tika/eval/app/reports/XLSXHREFFormatter.java | 12 +-
.../tika/eval/app/reports/XLSXNumFormatter.java | 9 +-
.../tika/eval/app/reports/XSLXCellFormatter.java | 3 +-
.../eval/app/tools/BatchTopCommonTokenCounter.java | 19 +-
.../eval/app/tools/CommonTokenOverlapCounter.java | 4 +-
.../apache/tika/eval/app/tools/LeipzigHelper.java | 21 +-
.../apache/tika/eval/app/tools/LeipzigSampler.java | 3 +-
.../eval/app/tools/SlowCompositeReaderWrapper.java | 64 +-
.../tika/eval/app/tools/TopCommonTokenCounter.java | 59 +-
.../apache/tika/eval/app/tools/TrainTestSplit.java | 21 +-
.../src/main/resources/comparison-reports-pg.xml | 3456 +++++++++----------
.../src/main/resources/comparison-reports-tags.xml | 3514 ++++++++++----------
.../tika-eval-app/src/main/resources/db.properties | 2 +-
.../src/main/resources/profile-reports-tags.xml | 608 ++--
.../src/main/resources/profile-reports.xml | 486 +--
.../main/resources/tika-eval-comparison-config.xml | 136 +-
.../resources/tika-eval-file-profiler-config.xml | 88 +-
.../main/resources/tika-eval-profiler-config.xml | 122 +-
.../apache/tika/eval/app/AnalyzerManagerTest.java | 7 +-
.../apache/tika/eval/app/ComparerBatchTest.java | 30 +-
.../apache/tika/eval/app/ProfilerBatchTest.java | 43 +-
.../apache/tika/eval/app/SimpleComparerTest.java | 102 +-
.../org/apache/tika/eval/app/TikaEvalCLITest.java | 87 +-
.../tika/eval/app/db/AbstractBufferTest.java | 14 +-
.../apache/tika/eval/app/io/ExtractReaderTest.java | 51 +-
.../tika/eval/app/io/FatalExceptionReaderTest.java | 3 +-
.../tika/eval/app/reports/ResultsReporterTest.java | 5 +-
.../eval/app/tools/TopCommonTokenCounterTest.java | 23 +-
.../single-file-profiler-crawl-extract-config.xml | 94 +-
.../single-file-profiler-crawl-input-config.xml | 96 +-
.../test-dirs/batch-logs/batch-process-fatal.xml | 9 +-
.../resources/test-dirs/extractsA/file1.pdf.json | 12 +-
.../test-dirs/extractsA/file12_es.txt.json | 10 +-
.../extractsA/file13_attachANotB.doc.json | 15 +-
.../extractsA/file14_diffAttachOrder.json | 1 -
.../test-dirs/extractsA/file2_attachANotB.doc.json | 13 +-
.../test-dirs/extractsA/file3_attachBNotA.doc.json | 10 +-
.../test-dirs/extractsA/file4_emptyB.pdf.json | 10 +-
.../test-dirs/extractsA/file6_accessEx.pdf.json | 24 +-
.../test-dirs/extractsA/file7_badJson.pdf.json | 11 +-
.../test-dirs/extractsA/file8_IOEx.pdf.json | 23 +-
.../resources/test-dirs/extractsB/file1.pdf.json | 8 +-
.../test-dirs/extractsB/file12_es.txt.json | 10 +-
.../extractsB/file14_diffAttachOrder.json | 1 -
.../resources/test-dirs/extractsB/file15_tags.html | 98 +-
.../test-dirs/extractsB/file16_badTags.html | 98 +-
.../test-dirs/extractsB/file2_attachANotB.doc.json | 10 +-
.../test-dirs/extractsB/file3_attachBNotA.doc.json | 13 +-
.../test-dirs/extractsB/file5_emptyA.pdf.json | 10 +-
.../test-dirs/extractsB/file6_accessEx.pdf.json | 24 +-
.../test-dirs/extractsB/file8_IOEx.pdf.json | 23 +-
.../apache/tika/example/ContentHandlerExample.java | 6 +-
.../org/apache/tika/example/CustomMimeInfo.java | 3 +-
.../org/apache/tika/example/DirListParser.java | 21 +-
.../apache/tika/example/DumpTikaConfigExample.java | 3 +-
.../example/EncryptedPrescriptionDetector.java | 3 +-
.../tika/example/EncryptedPrescriptionParser.java | 3 +-
.../apache/tika/example/ExtractEmbeddedFiles.java | 14 +-
.../tika/example/GrabPhoneNumbersExample.java | 9 +-
.../org/apache/tika/example/ImportContextImpl.java | 6 +-
.../tika/example/InterruptableParsingExample.java | 8 +-
.../java/org/apache/tika/example/Language.java | 3 +-
.../tika/example/LanguageDetectingParser.java | 3 +-
.../apache/tika/example/LuceneIndexerExtended.java | 3 +-
.../tika/example/MetadataAwareLuceneIndexer.java | 9 +-
.../java/org/apache/tika/example/MyFirstTika.java | 20 +-
.../org/apache/tika/example/ParsingExample.java | 28 +-
.../tika/example/PickBestTextEncodingParser.java | 17 +-
.../apache/tika/example/PrescriptionParser.java | 9 +-
.../java/org/apache/tika/example/RecentFiles.java | 22 +-
.../org/apache/tika/example/RollbackSoftware.java | 26 +-
.../org/apache/tika/example/SpringExample.java | 7 +-
.../tika/example/StandardsExtractionExample.java | 15 +-
.../org/apache/tika/example/TIAParsingExample.java | 10 +-
.../tika/example/TranscribeTranslateExample.java | 52 +-
.../apache/tika/example/TrecDocumentGenerator.java | 6 +-
.../resources/org/apache/tika/example/spring.xml | 16 +-
.../tika/example/ContentHandlerExampleTest.java | 19 +-
.../tika/example/DumpTikaConfigExampleTest.java | 24 +-
.../tika/example/SimpleTextExtractorTest.java | 11 +-
.../tika/example/SimpleTypeDetectorTest.java | 4 +-
.../apache/tika/example/TestParsingExample.java | 22 +-
.../src/test/resources/junit-platform.properties | 6 +-
.../org/apache/tika/fuzzing/cli/FuzzingCLI.java | 4 +-
.../pipes/xsearch/tests/TikaPipesXSearchBase.java | 8 +-
.../tika/pipes/s3/tests/PipeIntegrationTests.java | 8 +-
tika-parent/pom.xml | 134 +-
.../tika-parsers-ml/tika-age-recogniser/pom.xml | 4 +-
.../tika-parsers-ml/tika-parser-nlp-module/pom.xml | 2 +-
.../tika/parser/ner/NamedEntityParserTest.java | 11 +-
.../apache/tika/parser/crypto/TSDParserTest.java | 2 +-
.../org/apache/tika/parser/html/JSoupParser.java | 18 +
.../tika/parser/mail/MailContentHandler.java | 8 +-
.../apache/tika/parser/mail/RFC822ParserTest.java | 4 +
.../tika-parser-microsoft-module/pom.xml | 7 +
.../parser/microsoft/AbstractPOIFSExtractor.java | 36 +
.../tika/parser/microsoft/OutlookExtractor.java | 2 +-
.../tika/parser/microsoft/libpst/EmailVisitor.java | 100 +
.../tika/parser/microsoft/libpst/LibPstParser.java | 216 ++
.../microsoft/libpst/LibPstParserConfig.java | 81 +
.../parser/microsoft/pst/OutlookPSTParser.java | 182 +-
.../parser/microsoft/pst/PSTMailItemParser.java | 269 ++
.../services/org.apache.tika.parser.Parser | 1 +
.../tika/parser/microsoft/OutlookParserTest.java | 7 +
.../parser/microsoft/libpst/TestLibPstParser.java | 116 +
.../parser/microsoft/pst/OutlookPSTParserTest.java | 95 +-
.../microsoft/libpst/tika-libpst-config.xml} | 6 +-
.../microsoft/libpst/tika-libpst-eml-config.xml} | 7 +-
.../apache/tika/parser/ocr/TesseractOCRConfig.java | 10 +
.../apache/tika/parser/ocr/TesseractOCRParser.java | 36 +-
.../org/apache/tika/parser/pdf/XFAExtractor.java | 3 +-
.../org/apache/tika/parser/pkg/PackageParser.java | 21 +-
.../detect/zip/DefaultZipContainerDetector.java | 3 +-
.../microsoft/POIContainerExtractionTest.java | 2 +-
.../tika/parser/ocr/TesseractOCRParserTest.java | 18 +
tika-pipes/pom.xml | 4 +-
.../org/apache/tika/async/cli/TikaAsyncCLI.java | 5 +-
.../apache/tika/async/cli/AsyncProcessorTest.java | 47 +-
.../apache/tika/async/cli/TikaAsyncCLITest.java | 12 +-
.../tika/pipes/emitter/azblob/AZBlobEmitter.java | 36 +-
.../pipes/emitter/azblob/TestAZBlobEmitter.java | 8 +-
.../tika/pipes/emitter/fs/FileSystemEmitter.java | 12 +-
.../apache/tika/pipes/emitter/gcs/GCSEmitter.java | 37 +-
.../tika/pipes/emitter/gcs/TestGCSEmitter.java | 8 +-
.../tika/pipes/emitter/jdbc/JDBCEmitter.java | 5 +-
.../tika/pipes/emitter/jdbc/JDBCEmitterTest.java | 11 +-
.../tika/pipes/emitter/kafka/KafkaEmitter.java | 3 +-
.../emitter/opensearch/OpenSearchEmitter.java | 3 +-
.../apache/tika/pipes/emitter/s3/S3Emitter.java | 52 +-
.../tika/pipes/emitter/solr/SolrEmitter.java | 3 +-
.../pipes/emitter/solr/SolrEmitterDevTest.java | 3 +-
.../tika/pipes/fetcher/azblob/AZBlobFetcher.java | 28 +-
.../pipes/fetcher/azblob/TestAZBlobFetcher.java | 12 +-
.../apache/tika/pipes/fetcher/gcs/GCSFetcher.java | 3 +-
.../tika/pipes/fetcher/s3/TestGCSFetcher.java | 3 +-
tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml | 13 +
.../tika/pipes/fetcher/http/HttpFetcher.java | 94 +-
.../fetcher/http/config/AdditionalHttpHeaders.java | 41 +-
.../tika/pipes/fetcher/http/HttpFetcherTest.java | 63 +-
.../http/config/AdditionalHttpHeadersTest.java | 28 +-
.../microsoftgraph/MicrosoftGraphFetcher.java | 3 +-
.../microsoftgraph/MicrosoftGraphFetcherTest.java | 3 +-
.../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 5 +-
.../tika/pipes/fetcher/s3/TestS3Fetcher.java | 5 +-
.../tika-grpc/example-dockerfile/docker-build.sh | 6 +
tika-pipes/tika-grpc/pom.xml | 32 +-
.../tika/pipes/grpc/ExpiringFetcherStore.java | 25 +-
.../org/apache/tika/pipes/grpc/TikaGrpcServer.java | 3 +-
.../apache/tika/pipes/grpc/TikaGrpcServerImpl.java | 41 +-
tika-pipes/tika-grpc/src/main/proto/tika.proto | 67 +-
.../tika/pipes/grpc/ExpiringFetcherStoreTest.java | 30 +-
.../apache/tika/pipes/grpc/TikaGrpcServerTest.java | 1 +
tika-pipes/tika-pipes-iterators/pom.xml | 3 +-
.../tika-pipes-iterator-az-blob/pom.xml | 3 +-
.../pipesiterator/azblob/AZBlobPipesIterator.java | 40 +-
.../tika-pipes-iterator-csv/pom.xml | 3 +-
.../pipes/pipesiterator/csv/CSVPipesIterator.java | 47 +-
.../src/test/java/TestCSVPipesIterator.java | 17 +-
.../tika-pipes-iterator-gcs/pom.xml | 3 +-
.../pipes/pipesiterator/gcs/GCSPipesIterator.java | 19 +-
.../pipesiterator/gcs/TestGCSPipesIterator.java | 1 -
.../tika-pipes-iterator-jdbc/pom.xml | 3 +-
.../pipesiterator/jdbc/JDBCPipesIterator.java | 76 +-
.../pipesiterator/jdbc/TestJDBCPipesIterator.java | 75 +-
.../src/test/resources/log4j2.xml | 4 +-
.../tika-pipes-iterator-json/pom.xml | 3 +-
.../pipesiterator/json/JsonPipesIterator.java | 2 +-
.../pipesiterator/json/TestJsonPipesIterator.java | 21 +-
.../test-documents/test-with-embedded-bytes.json | 2200 +++++++++++-
.../src/test/resources/test-documents/test.json | 1500 ++++++++-
.../tika-pipes-iterator-kafka/pom.xml | 3 +-
.../pipesiterator/kafka/KafkaPipesIterator.java | 17 +-
.../tika-pipes-iterator-s3/pom.xml | 3 +-
.../pipes/pipesiterator/s3/S3PipesIterator.java | 33 +-
.../tika-pipes-iterator-solr/pom.xml | 3 +-
.../pipesiterator/solr/SolrPipesIterator.java | 41 +-
.../metadata/serialization/JsonFetchEmitTuple.java | 306 --
.../{metadata => }/serialization/JsonMetadata.java | 20 +-
.../serialization/JsonMetadataList.java | 28 +-
.../serialization/JsonStreamingSerializer.java | 12 +-
.../tika/serialization/ParseContextSerializer.java | 46 +
.../serialization/PrettyMetadataKeyComparator.java | 2 +-
.../tika/serialization/TikaJsonDeserializer.java | 401 +++
.../tika/serialization/TikaJsonSerializer.java | 282 ++
.../serialization/TikaSerializationException.java | 8 +-
.../pipes}/JsonEmitData.java | 16 +-
.../serialization/pipes/JsonFetchEmitTuple.java | 229 ++
.../pipes}/JsonFetchEmitTupleList.java | 38 +-
.../serialization/JsonMetadataListTest.java | 29 +-
.../serialization/JsonMetadataTest.java | 19 +-
.../serialization/TikaJsonSerializationTest.java | 37 +-
.../apache/tika/serialization/mocks/ClassA.java | 156 +
.../apache/tika/serialization/mocks/ClassB.java | 96 +
.../apache/tika/serialization/mocks/ClassC.java | 32 +-
.../pipes}/JsonFetchEmitTupleListTest.java | 6 +-
.../pipes}/JsonFetchEmitTupleTest.java | 48 +-
.../tika/server/client/TikaAsyncHttpClient.java | 7 +-
.../org/apache/tika/server/client/TikaClient.java | 5 +-
.../apache/tika/server/client/TikaClientCLI.java | 42 +-
.../tika/server/client/TikaEmitterResult.java | 3 +-
.../tika/server/client/TikaPipesHttpClient.java | 21 +-
.../tika/server/client/TikaServerClientConfig.java | 41 +-
.../org/apache/tika/server/client/TestBasic.java | 14 +-
tika-server/tika-server-core/pom.xml | 2 +-
.../server/core/CompositeParseContextConfig.java | 7 +-
.../server/core/DefaultInputStreamFactory.java | 3 +-
.../tika/server/core/FetcherStreamFactory.java | 63 +-
.../tika/server/core/InputStreamFactory.java | 4 +-
.../server/core/ProduceTypeResourceComparator.java | 54 +-
.../org/apache/tika/server/core/ServerStatus.java | 14 +-
.../tika/server/core/ServerStatusWatcher.java | 63 +-
.../org/apache/tika/server/core/TaskStatus.java | 3 +-
.../apache/tika/server/core/TikaLoggingFilter.java | 5 +-
.../org/apache/tika/server/core/TikaServerCli.java | 26 +-
.../apache/tika/server/core/TikaServerConfig.java | 71 +-
.../core/TikaServerParseExceptionMapper.java | 19 +-
.../apache/tika/server/core/TikaServerProcess.java | 191 +-
.../tika/server/core/TikaServerWatchDog.java | 84 +-
.../org/apache/tika/server/core/TlsConfig.java | 28 +-
.../apache/tika/server/core/WatchDogResult.java | 3 +-
.../server/core/config/DocumentSelectorConfig.java | 3 +-
.../server/core/config/PasswordProviderConfig.java | 3 +-
.../tika/server/core/config/TimeoutConfig.java | 3 +-
.../tika/server/core/resource/AsyncResource.java | 60 +-
.../server/core/resource/DetectorResource.java | 15 +-
.../server/core/resource/LanguageResource.java | 8 +-
.../server/core/resource/MetadataResource.java | 39 +-
.../tika/server/core/resource/PipesResource.java | 23 +-
.../core/resource/RecursiveMetadataResource.java | 56 +-
.../tika/server/core/resource/TikaDetectors.java | 24 +-
.../tika/server/core/resource/TikaMimeTypes.java | 197 +-
.../tika/server/core/resource/TikaParsers.java | 45 +-
.../tika/server/core/resource/TikaResource.java | 218 +-
.../tika/server/core/resource/TikaWelcome.java | 23 +-
.../server/core/resource/TranslateResource.java | 25 +-
.../server/core/resource/UnpackerResource.java | 82 +-
.../server/core/writer/CSVMessageBodyWriter.java | 15 +-
.../server/core/writer/JSONMessageBodyWriter.java | 14 +-
.../tika/server/core/writer/JSONObjWriter.java | 16 +-
.../core/writer/MetadataListMessageBodyWriter.java | 14 +-
.../apache/tika/server/core/writer/TarWriter.java | 15 +-
.../server/core/writer/TextMessageBodyWriter.java | 12 +-
.../apache/tika/server/core/writer/ZipWriter.java | 19 +-
.../org/apache/tika/server/core/CXFTestBase.java | 71 +-
.../tika/server/core/IntegrationTestBase.java | 29 +-
.../tika/server/core/LanguageResourceTest.java | 27 +-
.../server/core/RecursiveMetadataResourceTest.java | 15 +-
.../apache/tika/server/core/ServerStatusTest.java | 7 +-
.../apache/tika/server/core/StackTraceOffTest.java | 29 +-
.../apache/tika/server/core/StackTraceTest.java | 29 +-
.../apache/tika/server/core/TikaMimeTypesTest.java | 27 +-
.../org/apache/tika/server/core/TikaPipesTest.java | 131 +-
.../tika/server/core/TikaResourceFetcherTest.java | 48 +-
.../core/TikaResourceMetadataFilterTest.java | 14 +-
.../tika/server/core/TikaResourceNoStackTest.java | 20 +-
.../apache/tika/server/core/TikaResourceTest.java | 88 +-
.../core/TikaServerAsyncIntegrationTest.java | 72 +-
.../tika/server/core/TikaServerConfigTest.java | 78 +-
.../server/core/TikaServerIntegrationTest.java | 258 +-
.../core/TikaServerPipesIntegrationTest.java | 136 +-
.../tika/server/core/TikaServerStatusTest.java | 27 +-
.../apache/tika/server/core/TikaVersionTest.java | 11 +-
.../apache/tika/server/core/TikaWelcomeTest.java | 24 +-
.../tika/server/core/TranslateResourceTest.java | 21 +-
.../server/standard/config/PDFServerConfig.java | 12 +-
.../standard/config/TesseractServerConfig.java | 9 +-
.../standard/resource/XMPMetadataResource.java | 22 +-
.../standard/writer/XMPMessageBodyWriter.java | 12 +-
.../tika/server/standard/DetectorResourceTest.java | 28 +-
.../apache/tika/server/standard/FetcherTest.java | 20 +-
.../server/standard/JsonMaxFieldLengthTest.java | 25 +-
.../tika/server/standard/MetadataResourceTest.java | 101 +-
.../server/standard/OpenNLPMetadataFilterTest.java | 45 +-
.../standard/OptimaizeMetadataFilterTest.java | 48 +-
.../standard/RecursiveMetadataFilterTest.java | 20 +-
.../standard/RecursiveMetadataResourceTest.java | 320 +-
.../tika/server/standard/TikaDetectorsTest.java | 45 +-
.../tika/server/standard/TikaMimeTypesTest.java | 21 +-
.../tika/server/standard/TikaParsersTest.java | 40 +-
.../apache/tika/server/standard/TikaPipesTest.java | 122 +-
.../tika/server/standard/TikaResourceTest.java | 460 +--
.../tika/server/standard/UnpackerResourceTest.java | 132 +-
.../standard/UnpackerResourceWithConfigTest.java | 75 +-
.../test/resources/config/TIKA-3137-include.xml | 38 +-
.../config/tika-config-for-server-tests.xml | 20 +-
.../tika-config-langdetect-opennlp-filter.xml | 26 +-
.../tika-config-langdetect-optimaize-filter.xml | 26 +-
.../test/resources/config/tika-config-unpacker.xml | 26 +-
.../resources/config/tika-config-url-fetcher.xml | 20 +-
tika-translate/pom.xml | 2 +-
408 files changed, 16131 insertions(+), 10213 deletions(-)
copy tika-core/src/main/java/org/apache/tika/{sax/ContentHandlerFactory.java => extractor/ParentContentHandler.java} (65%)
copy tika-core/src/main/java/org/apache/tika/metadata/{Rendering.java => PST.java} (73%)
create mode 100644 tika-core/src/main/java/org/apache/tika/metadata/filter/ClearByAttachmentTypeMetadataFilter.java
copy tika-core/src/test/resources/org/apache/tika/config/{TIKA-3137-exclude.xml => TIKA-4261-clear-by-embedded-type.xml} (83%)
create mode 100644 tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/libpst/EmailVisitor.java
create mode 100644 tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/libpst/LibPstParser.java
create mode 100644 tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/libpst/LibPstParserConfig.java
create mode 100644 tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/pst/PSTMailItemParser.java
create mode 100644 tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/libpst/TestLibPstParser.java
copy tika-parsers/tika-parsers-standard/{tika-parsers-standard-package/src/test/resources/org/apache/tika/config/TIKA-1558-excludesub.xml => tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/org/apache/tika/parser/microsoft/libpst/tika-libpst-config.xml} (78%)
copy tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/{tika-parser-pkg-module/src/test/resources/org/apache/tika/parser/pkg/tika-gzip-config.xml => tika-parser-microsoft-module/src/test/resources/org/apache/tika/parser/microsoft/libpst/tika-libpst-eml-config.xml} (75%)
copy tika-core/src/main/java/org/apache/tika/renderer/PageRangeRequest.java => tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/AdditionalHttpHeaders.java (53%)
copy tika-example/src/test/java/org/apache/tika/example/LanguageDetectorExampleTest.java => tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/config/AdditionalHttpHeadersTest.java (55%)
delete mode 100644 tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
rename tika-serialization/src/main/java/org/apache/tika/{metadata => }/serialization/JsonMetadata.java (88%)
rename tika-serialization/src/main/java/org/apache/tika/{metadata => }/serialization/JsonMetadataList.java (83%)
rename tika-serialization/src/main/java/org/apache/tika/{metadata => }/serialization/JsonStreamingSerializer.java (86%)
create mode 100644 tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextSerializer.java
rename tika-serialization/src/main/java/org/apache/tika/{metadata => }/serialization/PrettyMetadataKeyComparator.java (96%)
create mode 100644 tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonDeserializer.java
create mode 100644 tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonSerializer.java
copy tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientConfigException.java => tika-serialization/src/main/java/org/apache/tika/serialization/TikaSerializationException.java (80%)
rename tika-serialization/src/main/java/org/apache/tika/{metadata/serialization => serialization/pipes}/JsonEmitData.java (78%)
create mode 100644 tika-serialization/src/main/java/org/apache/tika/serialization/pipes/JsonFetchEmitTuple.java
rename tika-serialization/src/main/java/org/apache/tika/{metadata/serialization => serialization/pipes}/JsonFetchEmitTupleList.java (64%)
rename tika-serialization/src/test/java/org/apache/tika/{metadata => }/serialization/JsonMetadataListTest.java (86%)
rename tika-serialization/src/test/java/org/apache/tika/{metadata => }/serialization/JsonMetadataTest.java (87%)
copy tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/chm/TestPmgiHeader.java => tika-serialization/src/test/java/org/apache/tika/serialization/TikaJsonSerializationTest.java (52%)
create mode 100644 tika-serialization/src/test/java/org/apache/tika/serialization/mocks/ClassA.java
create mode 100644 tika-serialization/src/test/java/org/apache/tika/serialization/mocks/ClassB.java
copy tika-core/src/main/java/org/apache/tika/renderer/PageRangeRequest.java => tika-serialization/src/test/java/org/apache/tika/serialization/mocks/ClassC.java (63%)
rename tika-serialization/src/test/java/org/apache/tika/{metadata/serialization => serialization/pipes}/JsonFetchEmitTupleListTest.java (90%)
rename tika-serialization/src/test/java/org/apache/tika/{metadata/serialization => serialization/pipes}/JsonFetchEmitTupleTest.java (64%)
(tika) 01/02: Merge branch 'TIKA-4181-grpc' into tika-grpc-3x-features
Posted by nd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ndipiazza pushed a commit to branch tika-grpc-3x-features
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 63583e0254a85fabd715bdd001472ec61f8950f0
Merge: e50944daf c520d0244
Author: Nicholas DiPiazza <nd...@apache.org>
AuthorDate: Sun Jun 23 14:54:48 2024 -0500
Merge branch 'TIKA-4181-grpc' into tika-grpc-3x-features
CHANGES.txt | 14 +
tika-app/pom.xml | 2 +-
.../batch/DigestingAutoDetectParserFactory.java | 9 +-
.../batch/builders/AppParserFactoryBuilder.java | 17 +-
.../apache/tika/cli/BatchCommandLineBuilder.java | 25 +-
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 258 +-
.../apache/tika/gui/ParsingTransferHandler.java | 9 +-
.../src/main/java/org/apache/tika/gui/TikaGUI.java | 78 +-
.../java/org/apache/tika/cli/TikaCLIAsyncTest.java | 36 +-
.../tika/cli/TikaCLIBatchCommandLineTest.java | 24 +-
.../tika/cli/TikaCLIBatchIntegrationTest.java | 65 +-
.../test/java/org/apache/tika/cli/TikaCLITest.java | 126 +-
.../tika/extractor/TestEmbeddedDocumentUtil.java | 8 +-
.../java/org/apache/tika/batch/BatchProcess.java | 111 +-
.../apache/tika/batch/BatchProcessDriverCLI.java | 44 +-
.../apache/tika/batch/FileResourceConsumer.java | 52 +-
.../org/apache/tika/batch/FileResourceCrawler.java | 20 +-
.../tika/batch/ParallelFileProcessingResult.java | 10 +-
.../java/org/apache/tika/batch/StatusReporter.java | 25 +-
.../batch/builders/AbstractConsumersBuilder.java | 7 +-
.../tika/batch/builders/BatchProcessBuilder.java | 64 +-
.../batch/builders/CommandLineParserBuilder.java | 20 +-
.../DefaultContentHandlerFactoryBuilder.java | 3 +-
.../tika/batch/builders/ICrawlerBuilder.java | 3 +-
.../tika/batch/builders/InterrupterBuilder.java | 3 +-
.../builders/ObjectFromDOMAndQueueBuilder.java | 3 +-
.../tika/batch/builders/ParserFactoryBuilder.java | 7 +-
.../batch/builders/SimpleLogReporterBuilder.java | 6 +-
.../tika/batch/builders/StatusReporterBuilder.java | 3 +-
.../apache/tika/batch/fs/AbstractFSConsumer.java | 7 +-
.../apache/tika/batch/fs/BasicTikaFSConsumer.java | 16 +-
.../apache/tika/batch/fs/FSBatchProcessCLI.java | 15 +-
.../apache/tika/batch/fs/FSDirectoryCrawler.java | 30 +-
.../apache/tika/batch/fs/FSDocumentSelector.java | 3 +-
.../org/apache/tika/batch/fs/FSFileResource.java | 20 +-
.../org/apache/tika/batch/fs/FSListCrawler.java | 7 +-
.../tika/batch/fs/FSOutputStreamFactory.java | 9 +-
.../org/apache/tika/batch/fs/FSProperties.java | 3 +-
.../main/java/org/apache/tika/batch/fs/FSUtil.java | 36 +-
.../batch/fs/RecursiveParserWrapperFSConsumer.java | 13 +-
.../tika/batch/fs/StreamOutRPWFSConsumer.java | 26 +-
.../fs/builders/BasicTikaFSConsumersBuilder.java | 73 +-
.../tika/batch/fs/builders/FSCrawlerBuilder.java | 39 +-
.../batch/fs/strawman/StrawManTikaAppDriver.java | 46 +-
.../java/org/apache/tika/util/ClassLoaderUtil.java | 10 +-
.../main/java/org/apache/tika/util/PropsUtil.java | 8 +-
.../main/java/org/apache/tika/util/XMLDOMUtil.java | 19 +-
.../RecursiveParserWrapperFSConsumerTest.java | 82 +-
.../org/apache/tika/batch/fs/BatchDriverTest.java | 25 +-
.../org/apache/tika/batch/fs/BatchProcessTest.java | 126 +-
.../org/apache/tika/batch/fs/FSBatchTestBase.java | 37 +-
.../apache/tika/batch/fs/FSFileResourceTest.java | 10 +-
.../tika/batch/mock/MockConsumersBuilder.java | 3 +-
.../tika/extractor/ParentContentHandler.java | 24 +-
.../main/java/org/apache/tika/metadata/Office.java | 8 +
.../StreamEmitter.java => metadata/PST.java} | 15 +-
.../apache/tika/metadata/TikaCoreProperties.java | 32 +-
.../ClearByAttachmentTypeMetadataFilter.java | 85 +
.../metadata/filter/CompositeMetadataFilter.java | 5 +
.../metadata/filter/FieldNameMappingFilter.java | 5 +
.../java/org/apache/tika/parser/ParseContext.java | 177 +-
.../apache/tika/parser/RecursiveParserWrapper.java | 102 +-
.../java/org/apache/tika/pipes/FetchEmitTuple.java | 91 +-
.../java/org/apache/tika/pipes/HandlerConfig.java | 63 +-
.../java/org/apache/tika/pipes/PipesClient.java | 5 +-
.../java/org/apache/tika/pipes/PipesServer.java | 58 +-
.../org/apache/tika/pipes/async/AsyncEmitter.java | 2 +-
.../apache/tika/pipes/emitter/AbstractEmitter.java | 6 +-
.../org/apache/tika/pipes/emitter/EmitData.java | 16 +-
.../org/apache/tika/pipes/emitter/EmitKey.java | 2 +
.../org/apache/tika/pipes/emitter/Emitter.java | 3 +-
.../apache/tika/pipes/emitter/EmptyEmitter.java | 3 +-
.../apache/tika/pipes/emitter/StreamEmitter.java | 3 +-
.../extractor/EmbeddedDocumentBytesConfig.java | 5 +-
.../EmittingEmbeddedDocumentBytesHandler.java | 17 +-
.../apache/tika/pipes/fetcher/EmptyFetcher.java | 3 +-
.../org/apache/tika/pipes/fetcher/Fetcher.java | 3 +-
.../apache/tika/pipes/fetcher/RangeFetcher.java | 8 +-
.../tika/pipes/fetcher/fs/FileSystemFetcher.java | 6 +-
.../apache/tika/pipes/fetcher/url/UrlFetcher.java | 3 +-
.../filelist/FileListPipesIterator.java | 6 +-
.../pipesiterator/fs/FileSystemPipesIterator.java | 6 +-
.../tika/sax/RecursiveParserWrapperHandler.java | 37 +-
.../java/org/apache/tika/utils/XMLReaderUtils.java | 59 +
.../org/apache/tika/mime/tika-mimetypes.xml | 25 +-
.../tika/metadata/filter/TestMetadataFilter.java | 19 +
.../org/apache/tika/mime/MimeDetectionTest.java | 23 +
.../org/apache/tika/parser/mock/MockParser.java | 3 +-
.../org/apache/tika/pipes/PipesClientTest.java | 3 +-
.../org/apache/tika/pipes/PipesServerTest.java | 18 +-
.../org/apache/tika/pipes/async/MockEmitter.java | 5 +-
.../org/apache/tika/pipes/async/MockFetcher.java | 3 +-
.../org/apache/tika/pipes/emitter/MockEmitter.java | 3 +-
.../org/apache/tika/pipes/fetcher/MockFetcher.java | 3 +-
.../config/TIKA-4261-clear-by-embedded-type.xml | 20 +-
.../org/apache/tika/eval/app/AbstractProfiler.java | 180 +-
.../org/apache/tika/eval/app/EvalFilePaths.java | 3 +-
.../org/apache/tika/eval/app/ExtractComparer.java | 187 +-
.../org/apache/tika/eval/app/ExtractProfiler.java | 152 +-
.../org/apache/tika/eval/app/FileProfiler.java | 48 +-
.../java/org/apache/tika/eval/app/TikaEvalCLI.java | 70 +-
.../apache/tika/eval/app/XMLErrorLogUpdater.java | 43 +-
.../tika/eval/app/batch/DBConsumersManager.java | 3 +-
.../tika/eval/app/batch/EvalConsumerBuilder.java | 23 +-
.../tika/eval/app/batch/EvalConsumersBuilder.java | 6 +-
.../eval/app/batch/ExtractComparerBuilder.java | 9 +-
.../eval/app/batch/ExtractProfilerBuilder.java | 7 +-
.../java/org/apache/tika/eval/app/db/Cols.java | 25 +-
.../java/org/apache/tika/eval/app/db/DBBuffer.java | 7 +-
.../java/org/apache/tika/eval/app/db/H2Util.java | 4 +-
.../java/org/apache/tika/eval/app/db/JDBCUtil.java | 29 +-
.../org/apache/tika/eval/app/db/MimeBuffer.java | 10 +-
.../java/org/apache/tika/eval/app/io/DBWriter.java | 13 +-
.../org/apache/tika/eval/app/io/ExtractReader.java | 40 +-
.../tika/eval/app/io/ExtractReaderException.java | 3 +-
.../org/apache/tika/eval/app/io/XMLLogReader.java | 3 +-
.../org/apache/tika/eval/app/reports/Report.java | 9 +-
.../tika/eval/app/reports/ResultsReporter.java | 80 +-
.../tika/eval/app/reports/XLSXHREFFormatter.java | 12 +-
.../tika/eval/app/reports/XLSXNumFormatter.java | 9 +-
.../tika/eval/app/reports/XSLXCellFormatter.java | 3 +-
.../eval/app/tools/BatchTopCommonTokenCounter.java | 19 +-
.../eval/app/tools/CommonTokenOverlapCounter.java | 4 +-
.../apache/tika/eval/app/tools/LeipzigHelper.java | 21 +-
.../apache/tika/eval/app/tools/LeipzigSampler.java | 3 +-
.../eval/app/tools/SlowCompositeReaderWrapper.java | 64 +-
.../tika/eval/app/tools/TopCommonTokenCounter.java | 59 +-
.../apache/tika/eval/app/tools/TrainTestSplit.java | 21 +-
.../src/main/resources/comparison-reports-pg.xml | 3456 +++++++++----------
.../src/main/resources/comparison-reports-tags.xml | 3514 ++++++++++----------
.../tika-eval-app/src/main/resources/db.properties | 2 +-
.../src/main/resources/profile-reports-tags.xml | 608 ++--
.../src/main/resources/profile-reports.xml | 486 +--
.../main/resources/tika-eval-comparison-config.xml | 136 +-
.../resources/tika-eval-file-profiler-config.xml | 88 +-
.../main/resources/tika-eval-profiler-config.xml | 122 +-
.../apache/tika/eval/app/AnalyzerManagerTest.java | 7 +-
.../apache/tika/eval/app/ComparerBatchTest.java | 30 +-
.../apache/tika/eval/app/ProfilerBatchTest.java | 43 +-
.../apache/tika/eval/app/SimpleComparerTest.java | 102 +-
.../org/apache/tika/eval/app/TikaEvalCLITest.java | 87 +-
.../tika/eval/app/db/AbstractBufferTest.java | 14 +-
.../apache/tika/eval/app/io/ExtractReaderTest.java | 51 +-
.../tika/eval/app/io/FatalExceptionReaderTest.java | 3 +-
.../tika/eval/app/reports/ResultsReporterTest.java | 5 +-
.../eval/app/tools/TopCommonTokenCounterTest.java | 23 +-
.../single-file-profiler-crawl-extract-config.xml | 94 +-
.../single-file-profiler-crawl-input-config.xml | 96 +-
.../test-dirs/batch-logs/batch-process-fatal.xml | 9 +-
.../resources/test-dirs/extractsA/file1.pdf.json | 12 +-
.../test-dirs/extractsA/file12_es.txt.json | 10 +-
.../extractsA/file13_attachANotB.doc.json | 15 +-
.../extractsA/file14_diffAttachOrder.json | 1 -
.../test-dirs/extractsA/file2_attachANotB.doc.json | 13 +-
.../test-dirs/extractsA/file3_attachBNotA.doc.json | 10 +-
.../test-dirs/extractsA/file4_emptyB.pdf.json | 10 +-
.../test-dirs/extractsA/file6_accessEx.pdf.json | 24 +-
.../test-dirs/extractsA/file7_badJson.pdf.json | 11 +-
.../test-dirs/extractsA/file8_IOEx.pdf.json | 23 +-
.../resources/test-dirs/extractsB/file1.pdf.json | 8 +-
.../test-dirs/extractsB/file12_es.txt.json | 10 +-
.../extractsB/file14_diffAttachOrder.json | 1 -
.../resources/test-dirs/extractsB/file15_tags.html | 98 +-
.../test-dirs/extractsB/file16_badTags.html | 98 +-
.../test-dirs/extractsB/file2_attachANotB.doc.json | 10 +-
.../test-dirs/extractsB/file3_attachBNotA.doc.json | 13 +-
.../test-dirs/extractsB/file5_emptyA.pdf.json | 10 +-
.../test-dirs/extractsB/file6_accessEx.pdf.json | 24 +-
.../test-dirs/extractsB/file8_IOEx.pdf.json | 23 +-
.../apache/tika/example/ContentHandlerExample.java | 6 +-
.../org/apache/tika/example/CustomMimeInfo.java | 3 +-
.../org/apache/tika/example/DirListParser.java | 21 +-
.../apache/tika/example/DumpTikaConfigExample.java | 3 +-
.../example/EncryptedPrescriptionDetector.java | 3 +-
.../tika/example/EncryptedPrescriptionParser.java | 3 +-
.../apache/tika/example/ExtractEmbeddedFiles.java | 14 +-
.../tika/example/GrabPhoneNumbersExample.java | 9 +-
.../org/apache/tika/example/ImportContextImpl.java | 6 +-
.../tika/example/InterruptableParsingExample.java | 8 +-
.../java/org/apache/tika/example/Language.java | 3 +-
.../tika/example/LanguageDetectingParser.java | 3 +-
.../apache/tika/example/LuceneIndexerExtended.java | 3 +-
.../tika/example/MetadataAwareLuceneIndexer.java | 9 +-
.../java/org/apache/tika/example/MyFirstTika.java | 20 +-
.../org/apache/tika/example/ParsingExample.java | 28 +-
.../tika/example/PickBestTextEncodingParser.java | 17 +-
.../apache/tika/example/PrescriptionParser.java | 9 +-
.../java/org/apache/tika/example/RecentFiles.java | 22 +-
.../org/apache/tika/example/RollbackSoftware.java | 26 +-
.../org/apache/tika/example/SpringExample.java | 7 +-
.../tika/example/StandardsExtractionExample.java | 15 +-
.../org/apache/tika/example/TIAParsingExample.java | 10 +-
.../tika/example/TranscribeTranslateExample.java | 52 +-
.../apache/tika/example/TrecDocumentGenerator.java | 6 +-
.../resources/org/apache/tika/example/spring.xml | 16 +-
.../tika/example/ContentHandlerExampleTest.java | 19 +-
.../tika/example/DumpTikaConfigExampleTest.java | 24 +-
.../tika/example/SimpleTextExtractorTest.java | 11 +-
.../tika/example/SimpleTypeDetectorTest.java | 4 +-
.../apache/tika/example/TestParsingExample.java | 22 +-
.../src/test/resources/junit-platform.properties | 6 +-
.../org/apache/tika/fuzzing/cli/FuzzingCLI.java | 4 +-
.../pipes/xsearch/tests/TikaPipesXSearchBase.java | 8 +-
.../tika/pipes/s3/tests/PipeIntegrationTests.java | 8 +-
tika-parent/pom.xml | 134 +-
.../tika-parsers-ml/tika-age-recogniser/pom.xml | 4 +-
.../tika-parsers-ml/tika-parser-nlp-module/pom.xml | 2 +-
.../tika/parser/ner/NamedEntityParserTest.java | 11 +-
.../apache/tika/parser/crypto/TSDParserTest.java | 2 +-
.../org/apache/tika/parser/html/JSoupParser.java | 18 +
.../tika/parser/mail/MailContentHandler.java | 8 +-
.../apache/tika/parser/mail/RFC822ParserTest.java | 4 +
.../tika-parser-microsoft-module/pom.xml | 7 +
.../parser/microsoft/AbstractPOIFSExtractor.java | 36 +
.../tika/parser/microsoft/OutlookExtractor.java | 2 +-
.../tika/parser/microsoft/libpst/EmailVisitor.java | 100 +
.../tika/parser/microsoft/libpst/LibPstParser.java | 216 ++
.../microsoft/libpst/LibPstParserConfig.java | 81 +
.../parser/microsoft/pst/OutlookPSTParser.java | 182 +-
.../parser/microsoft/pst/PSTMailItemParser.java | 269 ++
.../services/org.apache.tika.parser.Parser | 1 +
.../tika/parser/microsoft/OutlookParserTest.java | 7 +
.../parser/microsoft/libpst/TestLibPstParser.java | 116 +
.../parser/microsoft/pst/OutlookPSTParserTest.java | 95 +-
.../parser/microsoft/libpst/tika-libpst-config.xml | 17 +-
.../microsoft/libpst/tika-libpst-eml-config.xml | 21 +-
.../apache/tika/parser/ocr/TesseractOCRConfig.java | 10 +
.../apache/tika/parser/ocr/TesseractOCRParser.java | 36 +-
.../org/apache/tika/parser/pdf/XFAExtractor.java | 3 +-
.../org/apache/tika/parser/pkg/PackageParser.java | 21 +-
.../detect/zip/DefaultZipContainerDetector.java | 3 +-
.../microsoft/POIContainerExtractionTest.java | 2 +-
.../tika/parser/ocr/TesseractOCRParserTest.java | 18 +
tika-pipes/pom.xml | 4 +-
.../org/apache/tika/async/cli/TikaAsyncCLI.java | 5 +-
.../apache/tika/async/cli/AsyncProcessorTest.java | 47 +-
.../apache/tika/async/cli/TikaAsyncCLITest.java | 12 +-
.../tika/pipes/emitter/azblob/AZBlobEmitter.java | 36 +-
.../pipes/emitter/azblob/TestAZBlobEmitter.java | 8 +-
.../tika/pipes/emitter/fs/FileSystemEmitter.java | 12 +-
.../apache/tika/pipes/emitter/gcs/GCSEmitter.java | 37 +-
.../tika/pipes/emitter/gcs/TestGCSEmitter.java | 8 +-
.../tika/pipes/emitter/jdbc/JDBCEmitter.java | 5 +-
.../tika/pipes/emitter/jdbc/JDBCEmitterTest.java | 11 +-
.../tika/pipes/emitter/kafka/KafkaEmitter.java | 3 +-
.../emitter/opensearch/OpenSearchEmitter.java | 3 +-
.../apache/tika/pipes/emitter/s3/S3Emitter.java | 52 +-
.../tika/pipes/emitter/solr/SolrEmitter.java | 3 +-
.../pipes/emitter/solr/SolrEmitterDevTest.java | 3 +-
.../tika/pipes/fetcher/azblob/AZBlobFetcher.java | 28 +-
.../pipes/fetcher/azblob/TestAZBlobFetcher.java | 12 +-
.../apache/tika/pipes/fetcher/gcs/GCSFetcher.java | 3 +-
.../tika/pipes/fetcher/s3/TestGCSFetcher.java | 3 +-
tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml | 13 +
.../tika/pipes/fetcher/http/HttpFetcher.java | 81 +-
.../fetcher/http/config/AdditionalHttpHeaders.java | 65 +
.../tika/pipes/fetcher/http/HttpFetcherTest.java | 66 +-
.../http/config/AdditionalHttpHeadersTest.java | 32 +-
.../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 5 +-
.../tika/pipes/fetcher/s3/TestS3Fetcher.java | 5 +-
tika-pipes/tika-grpc/pom.xml | 32 +-
.../tika/pipes/grpc/ExpiringFetcherStore.java | 25 +-
.../org/apache/tika/pipes/grpc/TikaGrpcServer.java | 3 +-
.../apache/tika/pipes/grpc/TikaGrpcServerImpl.java | 40 +-
tika-pipes/tika-grpc/src/main/proto/tika.proto | 67 +-
.../tika/pipes/grpc/ExpiringFetcherStoreTest.java | 30 +-
.../apache/tika/pipes/grpc/TikaGrpcServerTest.java | 1 +
tika-pipes/tika-pipes-iterators/pom.xml | 3 +-
.../tika-pipes-iterator-az-blob/pom.xml | 3 +-
.../pipesiterator/azblob/AZBlobPipesIterator.java | 40 +-
.../tika-pipes-iterator-csv/pom.xml | 3 +-
.../pipes/pipesiterator/csv/CSVPipesIterator.java | 47 +-
.../src/test/java/TestCSVPipesIterator.java | 17 +-
.../tika-pipes-iterator-gcs/pom.xml | 3 +-
.../pipes/pipesiterator/gcs/GCSPipesIterator.java | 19 +-
.../pipesiterator/gcs/TestGCSPipesIterator.java | 1 -
.../tika-pipes-iterator-jdbc/pom.xml | 3 +-
.../pipesiterator/jdbc/JDBCPipesIterator.java | 76 +-
.../pipesiterator/jdbc/TestJDBCPipesIterator.java | 75 +-
.../src/test/resources/log4j2.xml | 4 +-
.../tika-pipes-iterator-json/pom.xml | 3 +-
.../pipesiterator/json/JsonPipesIterator.java | 2 +-
.../pipesiterator/json/TestJsonPipesIterator.java | 21 +-
.../test-documents/test-with-embedded-bytes.json | 2200 +++++++++++-
.../src/test/resources/test-documents/test.json | 1500 ++++++++-
.../tika-pipes-iterator-kafka/pom.xml | 3 +-
.../pipesiterator/kafka/KafkaPipesIterator.java | 17 +-
.../tika-pipes-iterator-s3/pom.xml | 3 +-
.../pipes/pipesiterator/s3/S3PipesIterator.java | 33 +-
.../tika-pipes-iterator-solr/pom.xml | 3 +-
.../pipesiterator/solr/SolrPipesIterator.java | 41 +-
.../metadata/serialization/JsonFetchEmitTuple.java | 306 --
.../{metadata => }/serialization/JsonMetadata.java | 20 +-
.../serialization/JsonMetadataList.java | 28 +-
.../serialization/JsonStreamingSerializer.java | 12 +-
.../tika/serialization/ParseContextSerializer.java | 46 +
.../serialization/PrettyMetadataKeyComparator.java | 2 +-
.../tika/serialization/TikaJsonDeserializer.java | 401 +++
.../tika/serialization/TikaJsonSerializer.java | 282 ++
.../serialization/TikaSerializationException.java | 17 +-
.../pipes}/JsonEmitData.java | 16 +-
.../serialization/pipes/JsonFetchEmitTuple.java | 229 ++
.../pipes}/JsonFetchEmitTupleList.java | 38 +-
.../serialization/JsonMetadataListTest.java | 29 +-
.../serialization/JsonMetadataTest.java | 19 +-
.../serialization/TikaJsonSerializationTest.java | 49 +
.../apache/tika/serialization/mocks/ClassA.java | 156 +
.../apache/tika/serialization/mocks/ClassB.java | 96 +
.../apache/tika/serialization/mocks/ClassC.java | 35 +-
.../pipes}/JsonFetchEmitTupleListTest.java | 6 +-
.../pipes}/JsonFetchEmitTupleTest.java | 48 +-
.../tika/server/client/TikaAsyncHttpClient.java | 7 +-
.../org/apache/tika/server/client/TikaClient.java | 5 +-
.../apache/tika/server/client/TikaClientCLI.java | 42 +-
.../tika/server/client/TikaEmitterResult.java | 3 +-
.../tika/server/client/TikaPipesHttpClient.java | 21 +-
.../tika/server/client/TikaServerClientConfig.java | 41 +-
.../org/apache/tika/server/client/TestBasic.java | 14 +-
tika-server/tika-server-core/pom.xml | 2 +-
.../server/core/CompositeParseContextConfig.java | 7 +-
.../server/core/DefaultInputStreamFactory.java | 3 +-
.../tika/server/core/FetcherStreamFactory.java | 63 +-
.../tika/server/core/InputStreamFactory.java | 4 +-
.../server/core/ProduceTypeResourceComparator.java | 54 +-
.../org/apache/tika/server/core/ServerStatus.java | 14 +-
.../tika/server/core/ServerStatusWatcher.java | 63 +-
.../org/apache/tika/server/core/TaskStatus.java | 3 +-
.../apache/tika/server/core/TikaLoggingFilter.java | 5 +-
.../org/apache/tika/server/core/TikaServerCli.java | 26 +-
.../apache/tika/server/core/TikaServerConfig.java | 71 +-
.../core/TikaServerParseExceptionMapper.java | 19 +-
.../apache/tika/server/core/TikaServerProcess.java | 191 +-
.../tika/server/core/TikaServerWatchDog.java | 84 +-
.../org/apache/tika/server/core/TlsConfig.java | 28 +-
.../apache/tika/server/core/WatchDogResult.java | 3 +-
.../server/core/config/DocumentSelectorConfig.java | 3 +-
.../server/core/config/PasswordProviderConfig.java | 3 +-
.../tika/server/core/config/TimeoutConfig.java | 3 +-
.../tika/server/core/resource/AsyncResource.java | 60 +-
.../server/core/resource/DetectorResource.java | 15 +-
.../server/core/resource/LanguageResource.java | 8 +-
.../server/core/resource/MetadataResource.java | 39 +-
.../tika/server/core/resource/PipesResource.java | 23 +-
.../core/resource/RecursiveMetadataResource.java | 56 +-
.../tika/server/core/resource/TikaDetectors.java | 24 +-
.../tika/server/core/resource/TikaMimeTypes.java | 197 +-
.../tika/server/core/resource/TikaParsers.java | 45 +-
.../tika/server/core/resource/TikaResource.java | 218 +-
.../tika/server/core/resource/TikaWelcome.java | 23 +-
.../server/core/resource/TranslateResource.java | 25 +-
.../server/core/resource/UnpackerResource.java | 82 +-
.../server/core/writer/CSVMessageBodyWriter.java | 15 +-
.../server/core/writer/JSONMessageBodyWriter.java | 14 +-
.../tika/server/core/writer/JSONObjWriter.java | 16 +-
.../core/writer/MetadataListMessageBodyWriter.java | 14 +-
.../apache/tika/server/core/writer/TarWriter.java | 15 +-
.../server/core/writer/TextMessageBodyWriter.java | 12 +-
.../apache/tika/server/core/writer/ZipWriter.java | 19 +-
.../org/apache/tika/server/core/CXFTestBase.java | 71 +-
.../tika/server/core/IntegrationTestBase.java | 29 +-
.../tika/server/core/LanguageResourceTest.java | 27 +-
.../server/core/RecursiveMetadataResourceTest.java | 15 +-
.../apache/tika/server/core/ServerStatusTest.java | 7 +-
.../apache/tika/server/core/StackTraceOffTest.java | 29 +-
.../apache/tika/server/core/StackTraceTest.java | 29 +-
.../apache/tika/server/core/TikaMimeTypesTest.java | 27 +-
.../org/apache/tika/server/core/TikaPipesTest.java | 131 +-
.../tika/server/core/TikaResourceFetcherTest.java | 48 +-
.../core/TikaResourceMetadataFilterTest.java | 14 +-
.../tika/server/core/TikaResourceNoStackTest.java | 20 +-
.../apache/tika/server/core/TikaResourceTest.java | 88 +-
.../core/TikaServerAsyncIntegrationTest.java | 72 +-
.../tika/server/core/TikaServerConfigTest.java | 78 +-
.../server/core/TikaServerIntegrationTest.java | 258 +-
.../core/TikaServerPipesIntegrationTest.java | 136 +-
.../tika/server/core/TikaServerStatusTest.java | 27 +-
.../apache/tika/server/core/TikaVersionTest.java | 11 +-
.../apache/tika/server/core/TikaWelcomeTest.java | 24 +-
.../tika/server/core/TranslateResourceTest.java | 21 +-
.../server/standard/config/PDFServerConfig.java | 12 +-
.../standard/config/TesseractServerConfig.java | 9 +-
.../standard/resource/XMPMetadataResource.java | 22 +-
.../standard/writer/XMPMessageBodyWriter.java | 12 +-
.../tika/server/standard/DetectorResourceTest.java | 28 +-
.../apache/tika/server/standard/FetcherTest.java | 20 +-
.../server/standard/JsonMaxFieldLengthTest.java | 25 +-
.../tika/server/standard/MetadataResourceTest.java | 101 +-
.../server/standard/OpenNLPMetadataFilterTest.java | 45 +-
.../standard/OptimaizeMetadataFilterTest.java | 48 +-
.../standard/RecursiveMetadataFilterTest.java | 20 +-
.../standard/RecursiveMetadataResourceTest.java | 320 +-
.../tika/server/standard/TikaDetectorsTest.java | 45 +-
.../tika/server/standard/TikaMimeTypesTest.java | 21 +-
.../tika/server/standard/TikaParsersTest.java | 40 +-
.../apache/tika/server/standard/TikaPipesTest.java | 122 +-
.../tika/server/standard/TikaResourceTest.java | 460 +--
.../tika/server/standard/UnpackerResourceTest.java | 132 +-
.../standard/UnpackerResourceWithConfigTest.java | 75 +-
.../test/resources/config/TIKA-3137-include.xml | 38 +-
.../config/tika-config-for-server-tests.xml | 20 +-
.../tika-config-langdetect-opennlp-filter.xml | 26 +-
.../tika-config-langdetect-optimaize-filter.xml | 26 +-
.../test/resources/config/tika-config-unpacker.xml | 26 +-
.../resources/config/tika-config-url-fetcher.xml | 20 +-
tika-translate/pom.xml | 2 +-
405 files changed, 16238 insertions(+), 10177 deletions(-)
diff --cc tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
index 59def749c,802545380..ce7225a9a
--- a/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
@@@ -69,11 -69,10 +69,12 @@@ import org.apache.tika.pipes.emitter.Em
import org.apache.tika.pipes.emitter.EmitterManager;
import org.apache.tika.pipes.emitter.StreamEmitter;
import org.apache.tika.pipes.emitter.TikaEmitterException;
+ import org.apache.tika.pipes.extractor.EmbeddedDocumentBytesConfig;
import org.apache.tika.pipes.extractor.EmittingEmbeddedDocumentBytesHandler;
+import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.pipes.fetcher.Fetcher;
import org.apache.tika.pipes.fetcher.FetcherManager;
+import org.apache.tika.pipes.fetcher.RangeFetcher;
import org.apache.tika.sax.BasicContentHandlerFactory;
import org.apache.tika.sax.ContentHandlerFactory;
import org.apache.tika.sax.RecursiveParserWrapperHandler;
@@@ -456,35 -458,18 +460,36 @@@ public class PipesServer implements Run
}
protected MetadataListAndEmbeddedBytes parseFromTuple(FetchEmitTuple t, Fetcher fetcher) {
-
- Metadata metadata = new Metadata();
- try (InputStream stream = fetcher.fetch(t.getFetchKey().getFetchKey(), metadata, t.getParseContext())) {
- return parseWithStream(t, stream, metadata);
- } catch (SecurityException e) {
- LOG.error("security exception " + t.getId(), e);
- throw e;
- } catch (TikaException | IOException e) {
- LOG.warn("fetch exception " + t.getId(), e);
- write(STATUS.FETCH_EXCEPTION, ExceptionUtils.getStackTrace(e));
+ FetchKey fetchKey = t.getFetchKey();
+ if (fetchKey.hasRange()) {
+ if (!(fetcher instanceof RangeFetcher)) {
+ throw new IllegalArgumentException(
+ "fetch key has a range, but the fetcher is not a range fetcher");
+ }
+ Metadata metadata = t.getMetadata() == null ? new Metadata() : t.getMetadata();
+ try (InputStream stream = ((RangeFetcher) fetcher).fetch(fetchKey.getFetchKey(),
+ fetchKey.getRangeStart(), fetchKey.getRangeEnd(), metadata)) {
+ return parseWithStream(t, stream, metadata);
+ } catch (SecurityException e) {
+ LOG.error("security exception " + t.getId(), e);
+ throw e;
+ } catch (TikaException | IOException e) {
+ LOG.warn("fetch exception " + t.getId(), e);
+ write(STATUS.FETCH_EXCEPTION, ExceptionUtils.getStackTrace(e));
+ }
+ } else {
+ Metadata metadata = t.getMetadata() == null ? new Metadata() : t.getMetadata();
- try (InputStream stream = fetcher.fetch(t.getFetchKey().getFetchKey(), metadata)) {
++ try (InputStream stream = fetcher.fetch(t.getFetchKey().getFetchKey(), metadata, t.getParseContext())) {
+ return parseWithStream(t, stream, metadata);
+ } catch (SecurityException e) {
+ LOG.error("security exception " + t.getId(), e);
+ throw e;
+ } catch (TikaException | IOException e) {
+ LOG.warn("fetch exception " + t.getId(), e);
+ write(STATUS.FETCH_EXCEPTION, ExceptionUtils.getStackTrace(e));
+ }
}
+
return null;
}
diff --cc tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
index c204370d1,c839ca6af..abbb84266
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
@@@ -45,11 -45,19 +45,24 @@@
<artifactId>tika-httpclient-commons</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.nimbusds</groupId>
+ <artifactId>nimbus-jose-jwt</artifactId>
+ <version>9.37.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-core</artifactId>
diff --cc tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
index 575e9bd7a,464f5960b..59e63675e
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
@@@ -68,12 -65,11 +68,14 @@@ import org.apache.tika.io.TikaInputStre
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
+ import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.fetcher.AbstractFetcher;
import org.apache.tika.pipes.fetcher.RangeFetcher;
+ import org.apache.tika.pipes.fetcher.http.config.AdditionalHttpHeaders;
import org.apache.tika.pipes.fetcher.http.config.HttpFetcherConfig;
+import org.apache.tika.pipes.fetcher.http.jwt.JwtGenerator;
+import org.apache.tika.pipes.fetcher.http.jwt.JwtPrivateKeyCreds;
+import org.apache.tika.pipes.fetcher.http.jwt.JwtSecretCreds;
import org.apache.tika.utils.StringUtils;
/**
@@@ -125,10 -121,8 +127,10 @@@ public class HttpFetcher extends Abstra
//back-off client that disables compression
private HttpClient noCompressHttpClient;
+ JwtGenerator jwtGenerator;
+
@Override
- public InputStream fetch(String fetchKey, Metadata metadata) throws IOException, TikaException {
+ public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws IOException {
HttpGet get = new HttpGet(fetchKey);
RequestConfig requestConfig = RequestConfig
.custom()
@@@ -140,56 -134,30 +142,47 @@@
return execute(get, metadata, httpClient, true);
}
- private void setHttpRequestHeaders(Metadata metadata, HttpGet get) throws TikaException {
- if (!StringUtils.isBlank(httpFetcherConfig.getUserAgent())) {
- get.setHeader(USER_AGENT, httpFetcherConfig.getUserAgent());
- }
+ @Override
+ public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata, ParseContext parseContext)
+ throws IOException {
+ HttpGet get = new HttpGet(fetchKey);
+ putAdditionalHeadersOnRequest(parseContext, get);
- // additional http request headers can be sent in here.
- // Add the headers from the Fetcher configuration.
- if (httpFetcherConfig.getHttpRequestHeaders() != null) {
- for (String httpRequestHeader : httpFetcherConfig.getHttpRequestHeaders()) {
- placeHeaderOnGetRequest(get, httpRequestHeader);
- }
+ get.setHeader("Range", "bytes=" + startRange + "-" + endRange);
+ return execute(get, metadata, httpClient, true);
+ }
+
+ private void putAdditionalHeadersOnRequest(ParseContext parseContext, HttpGet httpGet) {
+ if (!StringUtils.isBlank(httpFetcherConfig.getUserAgent())) {
+ httpGet.setHeader(USER_AGENT, httpFetcherConfig.getUserAgent());
}
- // Additionally, headers can be specified per-fetch via the metadata.
- String[] httpRequestHeaders = metadata.getValues("httpRequestHeaders");
- if (httpRequestHeaders != null) {
- for (String httpRequestHeader : httpRequestHeaders) {
- placeHeaderOnGetRequest(get, httpRequestHeader);
- }
+ AdditionalHttpHeaders additionalHttpHeaders = parseContext.get(AdditionalHttpHeaders.class);
+ if (additionalHttpHeaders != null) {
+ additionalHttpHeaders
+ .getHeaders()
+ .forEach(httpGet::setHeader);
}
+ if (jwtGenerator != null) {
+ try {
+ get.setHeader("Authorization", "Bearer " + jwtGenerator.jwt());
+ } catch (JOSEException e) {
+ throw new TikaException("Could not generate JWT", e);
+ }
+ }
+ }
+
+ private static void placeHeaderOnGetRequest(HttpGet get, String httpRequestHeader) {
- int idxOfEquals = httpRequestHeader.indexOf(':');
- if (idxOfEquals == -1) {
- return;
- }
- String headerKey = httpRequestHeader.substring(0, idxOfEquals).trim();
- String headerValue = httpRequestHeader.substring(idxOfEquals + 1).trim();
- get.setHeader(headerKey, headerValue);
- }
-
- @Override
- public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata) throws IOException, TikaException {
- HttpGet get = new HttpGet(fetchKey);
- if (!StringUtils.isBlank(httpFetcherConfig.getUserAgent())) {
- get.setHeader(USER_AGENT, httpFetcherConfig.getUserAgent());
- }
- setHttpRequestHeaders(metadata, get);
- get.setHeader("Range", "bytes=" + startRange + "-" + endRange);
- return execute(get, metadata, httpClient, true);
++ int idxOfEquals = httpRequestHeader.indexOf(':');
++ if (idxOfEquals == -1) {
++ return;
++ }
++ String headerKey = httpRequestHeader.substring(0, idxOfEquals).trim();
++ String headerValue = httpRequestHeader.substring(idxOfEquals + 1).trim();
++ get.setHeader(headerKey, headerValue);
}
- private InputStream execute(HttpGet get, Metadata metadata, HttpClient client, boolean retryOnBadLength) throws IOException {
+ private InputStream execute(HttpGet get, Metadata metadata, HttpClient client,
+ boolean retryOnBadLength) throws IOException {
HttpClientContext context = HttpClientContext.create();
HttpResponse response = null;
final AtomicBoolean timeout = new AtomicBoolean(false);
diff --cc tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java
index 56e83ea64,7d3dbdb7e..56253e3e2
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java
@@@ -30,7 -30,7 +30,8 @@@ import java.nio.file.Files
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
+import java.security.SecureRandom;
+ import java.util.ArrayList;
import java.util.Collections;
import java.util.zip.GZIPInputStream;
@@@ -58,12 -58,14 +59,15 @@@ import org.apache.tika.client.HttpClien
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
+ import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.fetcher.FetcherManager;
+ import org.apache.tika.pipes.fetcher.http.config.AdditionalHttpHeaders;
+ import org.apache.tika.pipes.fetcher.http.config.HttpFetcherConfig;
+import org.apache.tika.pipes.fetcher.http.jwt.JwtGenerator;
-public class HttpFetcherTest extends TikaTest {
-
+class HttpFetcherTest extends TikaTest {
private static final String TEST_URL = "wontbecalled";
private static final String CONTENT = "request content";
(tika) 02/02: resolve conflicts
Posted by nd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ndipiazza pushed a commit to branch tika-grpc-3x-features
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 4fb194e66d67cbb6e6044c864ed3ff3c2c218302
Author: Nicholas DiPiazza <nd...@apache.org>
AuthorDate: Sun Jun 23 14:55:14 2024 -0500
resolve conflicts
---
.../tika/pipes/fetcher/http/HttpFetcher.java | 51 +++++++-------
.../tika/pipes/fetcher/http/HttpFetcherTest.java | 77 +++++-----------------
.../microsoftgraph/MicrosoftGraphFetcher.java | 3 +-
.../microsoftgraph/MicrosoftGraphFetcherTest.java | 3 +-
.../tika-grpc/example-dockerfile/docker-build.sh | 6 ++
.../apache/tika/pipes/grpc/TikaGrpcServerImpl.java | 1 -
6 files changed, 55 insertions(+), 86 deletions(-)
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
index 59e63675e..649a2b0ea 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
@@ -130,7 +130,7 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
JwtGenerator jwtGenerator;
@Override
- public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws IOException {
+ public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws IOException, TikaException {
HttpGet get = new HttpGet(fetchKey);
RequestConfig requestConfig = RequestConfig
.custom()
@@ -143,8 +143,8 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
}
@Override
- public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata, ParseContext parseContext)
- throws IOException {
+ public InputStream fetch(String fetchKey, long startRange, long endRange, Metadata metadata,
+ ParseContext parseContext) throws IOException, TikaException {
HttpGet get = new HttpGet(fetchKey);
putAdditionalHeadersOnRequest(parseContext, get);
@@ -152,7 +152,7 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
return execute(get, metadata, httpClient, true);
}
- private void putAdditionalHeadersOnRequest(ParseContext parseContext, HttpGet httpGet) {
+ private void putAdditionalHeadersOnRequest(ParseContext parseContext, HttpGet httpGet) throws TikaException {
if (!StringUtils.isBlank(httpFetcherConfig.getUserAgent())) {
httpGet.setHeader(USER_AGENT, httpFetcherConfig.getUserAgent());
}
@@ -164,7 +164,7 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
}
if (jwtGenerator != null) {
try {
- get.setHeader("Authorization", "Bearer " + jwtGenerator.jwt());
+ httpGet.setHeader("Authorization", "Bearer " + jwtGenerator.jwt());
} catch (JOSEException e) {
throw new TikaException("Could not generate JWT", e);
}
@@ -172,17 +172,20 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
}
private static void placeHeaderOnGetRequest(HttpGet get, String httpRequestHeader) {
- int idxOfEquals = httpRequestHeader.indexOf(':');
- if (idxOfEquals == -1) {
- return;
- }
- String headerKey = httpRequestHeader.substring(0, idxOfEquals).trim();
- String headerValue = httpRequestHeader.substring(idxOfEquals + 1).trim();
- get.setHeader(headerKey, headerValue);
+ int idxOfEquals = httpRequestHeader.indexOf(':');
+ if (idxOfEquals == -1) {
+ return;
+ }
+ String headerKey = httpRequestHeader
+ .substring(0, idxOfEquals)
+ .trim();
+ String headerValue = httpRequestHeader
+ .substring(idxOfEquals + 1)
+ .trim();
+ get.setHeader(headerKey, headerValue);
}
- private InputStream execute(HttpGet get, Metadata metadata, HttpClient client,
- boolean retryOnBadLength) throws IOException {
+ private InputStream execute(HttpGet get, Metadata metadata, HttpClient client, boolean retryOnBadLength) throws IOException {
HttpClientContext context = HttpClientContext.create();
HttpResponse response = null;
final AtomicBoolean timeout = new AtomicBoolean(false);
@@ -228,7 +231,8 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
.contains("Premature " + "end of " + "Content-Length delimited message")) {
//one trigger for this is if the server sends the uncompressed length
//and then compresses the stream. See HTTPCLIENT-2176
- LOG.warn("premature end of content-length delimited message; retrying with " + "content compression disabled for {}", get.getURI());
+ LOG.warn("premature end of content-length delimited message; retrying with " + "content compression" +
+ " disabled for {}", get.getURI());
return execute(get, metadata, noCompressHttpClient, false);
}
throw e;
@@ -444,7 +448,9 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
public void setHttpRequestHeaders(List<String> headers) {
httpFetcherConfig.setHttpRequestHeaders(new ArrayList<>());
if (headers != null) {
- httpFetcherConfig.getHttpRequestHeaders().addAll(headers);
+ httpFetcherConfig
+ .getHttpRequestHeaders()
+ .addAll(headers);
}
}
@@ -458,7 +464,9 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
public void setHttpHeaders(List<String> headers) {
httpFetcherConfig.setHttpHeaders(new ArrayList<>());
if (headers != null) {
- httpFetcherConfig.getHttpHeaders().addAll(headers);
+ httpFetcherConfig
+ .getHttpHeaders()
+ .addAll(headers);
}
}
@@ -553,17 +561,16 @@ public class HttpFetcher extends AbstractFetcher implements Initializable, Range
jwtGenerator = new JwtGenerator(new JwtPrivateKeyCreds(key, httpFetcherConfig.getJwtIssuer(),
httpFetcherConfig.getJwtSubject(), httpFetcherConfig.getJwtExpiresInSeconds()));
} else if (!StringUtils.isBlank(httpFetcherConfig.getJwtSecret())) {
- jwtGenerator = new JwtGenerator(new JwtSecretCreds(httpFetcherConfig.getJwtSecret().getBytes(StandardCharsets.UTF_8),
- httpFetcherConfig.getJwtIssuer(),
- httpFetcherConfig.getJwtSubject(), httpFetcherConfig.getJwtExpiresInSeconds()));
+ jwtGenerator = new JwtGenerator(new JwtSecretCreds(httpFetcherConfig
+ .getJwtSecret()
+ .getBytes(StandardCharsets.UTF_8), httpFetcherConfig.getJwtIssuer(), httpFetcherConfig.getJwtSubject(), httpFetcherConfig.getJwtExpiresInSeconds()));
}
}
@Override
public void checkInitialization(InitializableProblemHandler problemHandler) throws TikaConfigException {
if (!StringUtils.isBlank(httpFetcherConfig.getJwtSecret()) && !StringUtils.isBlank(httpFetcherConfig.getJwtPrivateKeyBase64())) {
- throw new TikaConfigException("Both JWT secret and JWT private key base 64 were " +
- "specified. Only one or the other is supported");
+ throw new TikaConfigException("Both JWT secret and JWT private key base 64 were " + "specified. Only one or the other is supported");
}
}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java b/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java
index 56253e3e2..2e9f58e09 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/src/test/java/org/apache/tika/pipes/fetcher/http/HttpFetcherTest.java
@@ -63,7 +63,6 @@ import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.fetcher.FetcherManager;
-import org.apache.tika.pipes.fetcher.http.config.AdditionalHttpHeaders;
import org.apache.tika.pipes.fetcher.http.config.HttpFetcherConfig;
import org.apache.tika.pipes.fetcher.http.jwt.JwtGenerator;
@@ -90,8 +89,7 @@ class HttpFetcherTest extends TikaTest {
httpFetcherConfig.setOverallTimeout(400_000L);
httpFetcherConfig.setMaxSpoolSize(-1L);
- final HttpResponse mockResponse = buildMockResponse(HttpStatus.SC_OK,
- IOUtils.toInputStream(CONTENT, Charset.defaultCharset()));
+ final HttpResponse mockResponse = buildMockResponse(HttpStatus.SC_OK, IOUtils.toInputStream(CONTENT, Charset.defaultCharset()));
mockClientResponse(mockResponse);
}
@@ -136,7 +134,7 @@ class HttpFetcherTest extends TikaTest {
final Metadata meta = new Metadata();
meta.set(TikaCoreProperties.RESOURCE_NAME_KEY, "fileName");
- try (final InputStream ignored = httpFetcher.fetch(TEST_URL, meta)) {
+ try (final InputStream ignored = httpFetcher.fetch(TEST_URL, meta, new ParseContext())) {
// HTTP headers added into meta
assertEquals("200", meta.get("http-header:status-code"));
assertEquals(TEST_URL, meta.get("http-connection:target-url"));
@@ -147,7 +145,9 @@ class HttpFetcherTest extends TikaTest {
assertEquals("fileName", meta.get(TikaCoreProperties.RESOURCE_NAME_KEY));
}
- Mockito.verify(httpFetcher.jwtGenerator).jwt();
+ Mockito
+ .verify(httpFetcher.jwtGenerator)
+ .jwt();
}
@Test
@@ -157,8 +157,7 @@ class HttpFetcherTest extends TikaTest {
CloseableHttpResponse response = mock(CloseableHttpResponse.class);
ArgumentCaptor<HttpGet> httpGetArgumentCaptor = ArgumentCaptor.forClass(HttpGet.class);
- when(httpClient.execute(httpGetArgumentCaptor.capture(), any(HttpContext.class)))
- .thenReturn(response);
+ when(httpClient.execute(httpGetArgumentCaptor.capture(), any(HttpContext.class))).thenReturn(response);
when(response.getStatusLine()).thenReturn(new StatusLine() {
@Override
public ProtocolVersion getProtocolVersion() {
@@ -179,8 +178,8 @@ class HttpFetcherTest extends TikaTest {
when(response.getEntity()).thenReturn(new StringEntity("Hi"));
Metadata metadata = new Metadata();
- metadata.set(Property.externalText("httpRequestHeaders"), new String[] {"nick1: val1", "nick2: val2"});
- httpFetcher.fetch("http://localhost", metadata);
+ metadata.set(Property.externalText("httpRequestHeaders"), new String[]{"nick1: val1", "nick2: val2"});
+ httpFetcher.fetch("http://localhost", metadata, new ParseContext());
HttpGet httpGet = httpGetArgumentCaptor.getValue();
Assertions.assertEquals("val1", httpGet.getHeaders("nick1")[0].getValue());
Assertions.assertEquals("val2", httpGet.getHeaders("nick2")[0].getValue());
@@ -194,8 +193,7 @@ class HttpFetcherTest extends TikaTest {
String url = "https://t.co/cvfkWAEIxw?amp=1";
ByteArrayOutputStream bos = new ByteArrayOutputStream();
Metadata metadata = new Metadata();
- HttpFetcher httpFetcher =
- (HttpFetcher) getFetcherManager("tika-config-http.xml").getFetcher("http");
+ HttpFetcher httpFetcher = (HttpFetcher) getFetcherManager("tika-config-http.xml").getFetcher("http");
try (InputStream is = httpFetcher.fetch(url, metadata, new ParseContext())) {
IOUtils.copy(is, bos);
}
@@ -205,13 +203,11 @@ class HttpFetcherTest extends TikaTest {
@Test
@Disabled("requires network connectivity")
public void testRange() throws Exception {
- String url =
- "https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2020-45/segments/1603107869785.9/warc/CC-MAIN-20201020021700-20201020051700-00529.warc.gz";
+ String url = "https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2020-45/segments/1603107869785.9/warc/CC-MAIN-20201020021700-20201020051700-00529.warc.gz";
long start = 969596307;
long end = start + 1408 - 1;
Metadata metadata = new Metadata();
- HttpFetcher httpFetcher =
- (HttpFetcher) getFetcherManager("tika-config-http.xml").getFetcher("http");
+ HttpFetcher httpFetcher = (HttpFetcher) getFetcherManager("tika-config-http.xml").getFetcher("http");
try (TemporaryResources tmp = new TemporaryResources()) {
Path tmpPath = tmp.createTempFile(metadata);
try (InputStream is = httpFetcher.fetch(url, start, end, metadata)) {
@@ -221,49 +217,10 @@ class HttpFetcherTest extends TikaTest {
}
}
- @Test
- public void testHttpRequestHeaders() throws Exception {
- HttpClient httpClient = Mockito.mock(HttpClient.class);
- httpFetcher.setHttpClient(httpClient);
- CloseableHttpResponse response = mock(CloseableHttpResponse.class);
- ArgumentCaptor<HttpGet> httpGetArgumentCaptor = ArgumentCaptor.forClass(HttpGet.class);
-
- when(httpClient.execute(httpGetArgumentCaptor.capture(), any(HttpContext.class)))
- .thenReturn(response);
- when(response.getStatusLine()).thenReturn(new StatusLine() {
- @Override
- public ProtocolVersion getProtocolVersion() {
- return new HttpGet("http://localhost").getProtocolVersion();
- }
-
- @Override
- public int getStatusCode() {
- return 200;
- }
-
- @Override
- public String getReasonPhrase() {
- return null;
- }
- });
-
- when(response.getEntity()).thenReturn(new StringEntity("Hi"));
-
- Metadata metadata = new Metadata();
- ParseContext parseContext = new ParseContext();
- AdditionalHttpHeaders additionalHttpHeaders = new AdditionalHttpHeaders();
- additionalHttpHeaders.getHeaders().put("nick1", "val1");
- additionalHttpHeaders.getHeaders().put("nick2", "val2");
- parseContext.set(AdditionalHttpHeaders.class, additionalHttpHeaders);
- httpFetcher.fetch("http://localhost", metadata, parseContext);
- HttpGet httpGet = httpGetArgumentCaptor.getValue();
- Assertions.assertEquals("val1", httpGet.getHeaders("nick1")[0].getValue());
- Assertions.assertEquals("val2", httpGet.getHeaders("nick2")[0].getValue());
- }
-
FetcherManager getFetcherManager(String path) throws Exception {
- return FetcherManager.load(
- Paths.get(HttpFetcherTest.class.getResource("/" + path).toURI()));
+ return FetcherManager.load(Paths.get(HttpFetcherTest.class
+ .getResource("/" + path)
+ .toURI()));
}
private void mockClientResponse(final HttpResponse response) throws Exception {
@@ -272,8 +229,7 @@ class HttpFetcherTest extends TikaTest {
final HttpClient httpClient = mock(HttpClient.class);
final HttpClientFactory clientFactory = mock(HttpClientFactory.class);
- when(httpClient.execute(
- any(HttpUriRequest.class), any(HttpContext.class))).thenReturn(response);
+ when(httpClient.execute(any(HttpUriRequest.class), any(HttpContext.class))).thenReturn(response);
when(clientFactory.build()).thenReturn(httpClient);
when(clientFactory.copy()).thenReturn(clientFactory);
@@ -282,8 +238,7 @@ class HttpFetcherTest extends TikaTest {
httpFetcher.initialize(Collections.emptyMap());
}
- private static HttpResponse buildMockResponse(final int statusCode, final InputStream is)
- throws IOException {
+ private static HttpResponse buildMockResponse(final int statusCode, final InputStream is) throws IOException {
final HttpResponse response = mock(HttpResponse.class);
final StatusLine status = mock(StatusLine.class);
final HttpEntity entity = mock(HttpEntity.class);
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/java/org/apache/tika/pipes/fetchers/microsoftgraph/MicrosoftGraphFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/java/org/apache/tika/pipes/fetchers/microsoftgraph/MicrosoftGraphFetcher.java
index aae696ad2..61b486399 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/java/org/apache/tika/pipes/fetchers/microsoftgraph/MicrosoftGraphFetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/java/org/apache/tika/pipes/fetchers/microsoftgraph/MicrosoftGraphFetcher.java
@@ -34,6 +34,7 @@ import org.apache.tika.config.Param;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.fetcher.AbstractFetcher;
import org.apache.tika.pipes.fetchers.microsoftgraph.config.ClientCertificateCredentialsConfig;
import org.apache.tika.pipes.fetchers.microsoftgraph.config.ClientSecretCredentialsConfig;
@@ -109,7 +110,7 @@ public class MicrosoftGraphFetcher extends AbstractFetcher implements Initializa
}
@Override
- public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException {
+ public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws TikaException, IOException {
int tries = 0;
Exception ex;
do {
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/test/java/org/apache/tika/pipes/fetchers/microsoftgraph/MicrosoftGraphFetcherTest.java b/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/test/java/org/apache/tika/pipes/fetchers/microsoftgraph/MicrosoftGraphFetcherTest.java
index 0fafdfa47..6a2e71855 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/test/java/org/apache/tika/pipes/fetchers/microsoftgraph/MicrosoftGraphFetcherTest.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/test/java/org/apache/tika/pipes/fetchers/microsoftgraph/MicrosoftGraphFetcherTest.java
@@ -41,6 +41,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.fetchers.microsoftgraph.config.ClientCertificateCredentialsConfig;
import org.apache.tika.pipes.fetchers.microsoftgraph.config.MsGraphFetcherConfig;
@@ -95,7 +96,7 @@ class MicrosoftGraphFetcherTest {
Mockito.when(contentRequestBuilder.get())
.thenReturn(new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)));
InputStream resultingInputStream =
- microsoftGraphFetcher.fetch(siteDriveId + "," + driveItemid, new Metadata());
+ microsoftGraphFetcher.fetch(siteDriveId + "," + driveItemid, new Metadata(), new ParseContext());
Assertions.assertEquals(content,
IOUtils.toString(resultingInputStream, StandardCharsets.UTF_8));
}
diff --git a/tika-pipes/tika-grpc/example-dockerfile/docker-build.sh b/tika-pipes/tika-grpc/example-dockerfile/docker-build.sh
index a00f692b0..cb0df8d9f 100644
--- a/tika-pipes/tika-grpc/example-dockerfile/docker-build.sh
+++ b/tika-pipes/tika-grpc/example-dockerfile/docker-build.sh
@@ -15,6 +15,12 @@ rm -rf "${OUT_DIR}"
mkdir -p "${OUT_DIR}"
cp -r "${TIKA_SRC_PATH}/tika-pipes/tika-grpc/target/dependency" "${OUT_DIR}/libs"
+cp -r "${TIKA_SRC_PATH}/tika-pipes/tika-fetchers/tika-fetcher-gcs/target/tika-fetcher-gcs-"*".jar" "${OUT_DIR}/libs"
+cp -r "${TIKA_SRC_PATH}/tika-pipes/tika-fetchers/tika-fetcher-az-blob/target/tika-fetcher-az-blob-"*".jar" "${OUT_DIR}/libs"
+cp -r "${TIKA_SRC_PATH}/tika-pipes/tika-fetchers/tika-fetcher-http/target/tika-fetcher-http-"*".jar" "${OUT_DIR}/libs"
+cp -r "${TIKA_SRC_PATH}/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/target/tika-fetcher-microsoft-graph-"*".jar" "${OUT_DIR}/libs"
+cp -r "${TIKA_SRC_PATH}/tika-pipes/tika-fetchers/tika-fetcher-s3/target/tika-fetcher-s3-"*".jar" "${OUT_DIR}/libs"
+
cp "${TIKA_SRC_PATH}/tika-pipes/tika-grpc/target/tika-grpc-"*".jar" "${OUT_DIR}/libs"
cp "${TIKA_SRC_PATH}/tika-pipes/tika-grpc/src/test/resources/log4j2.xml" "${OUT_DIR}"
cp "${TIKA_SRC_PATH}/tika-pipes/tika-grpc/src/test/resources/tika-pipes-test-config.xml" "${OUT_DIR}/tika-config.xml"
diff --git a/tika-pipes/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java b/tika-pipes/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java
index a8e93aaad..42872e087 100644
--- a/tika-pipes/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java
+++ b/tika-pipes/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java
@@ -80,7 +80,6 @@ import org.apache.tika.pipes.emitter.EmitKey;
import org.apache.tika.pipes.fetcher.AbstractFetcher;
import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.pipes.fetcher.config.AbstractConfig;
-import org.apache.tika.utils.StringUtils;
class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase {
private static final Logger LOG = LoggerFactory.getLogger(TikaConfigSerializer.class);