You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by nd...@apache.org on 2026/08/13 22:38:58 UTC
(tika) branch pr-3018 created (now 6f07d05d0f)
This is an automated email from the ASF dual-hosted git repository.
nddipiazza pushed a change to branch pr-3018
in repository https://gitbox.apache.org/repos/asf/tika.git
at 6f07d05d0f Merge remote-tracking branch 'origin/main' into pr-3018
This branch includes the following new commits:
new 6f07d05d0f Merge remote-tracking branch 'origin/main' into pr-3018
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(tika) 01/01: Merge remote-tracking branch 'origin/main' into pr-3018
Posted by nd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nddipiazza pushed a commit to branch pr-3018
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 6f07d05d0fd8fcba9df8fc844283f008ee45d568
Merge: 177d21b9da 7d6fe60962
Author: nddipiazza <nd...@users.noreply.github.com>
AuthorDate: Thu Aug 13 17:38:30 2026 -0500
Merge remote-tracking branch 'origin/main' into pr-3018
.devcontainer/Dockerfile | 22 +
.../index.adoc => .devcontainer/devcontainer.json | 26 +-
.github/workflows/main-jdk17-build.yml | 5 +-
.skills/dev.md | 133 --
.skills/dev/SKILL.md | 210 +++
.../SKILL.md} | 34 +-
.skills/pr-review/SKILL.md | 211 +++
.../SKILL.md} | 31 +-
.../SKILL.md} | 25 +
.../SKILL.md} | 25 +
AGENTS.md | 34 +
CHANGES.txt | 117 ++
.../integration-testing/run-uat-script.adoc | 14 +-
.../advanced/integration-testing/tika-app.adoc | 2 +-
.../integration-testing/tika-eval-regression.adoc | 3 +-
.../advanced/integration-testing/tika-server.adoc | 26 +-
.../ROOT/pages/advanced/local-vlm-server.adoc | 6 +-
.../ROOT/pages/advanced/setting-limits.adoc | 68 +-
.../configuration/parsers/external-parser.adoc | 2 +-
.../pages/configuration/parsers/tess4j-parser.adoc | 9 +-
.../pages/configuration/parsers/vlm-parsers.adoc | 6 +-
docs/modules/ROOT/pages/developers/index.adoc | 2 +-
.../ROOT/pages/developers/metadata-keys.adoc | 43 +-
.../ROOT/pages/developers/serialization.adoc | 19 +-
.../pages/migration-to-4x/metadata-changes-4x.adoc | 226 +++-
.../migration-to-4x/migrating-tika-server-4x.adoc | 229 +++-
.../pages/migration-to-4x/migrating-to-4x.adoc | 44 +-
docs/modules/ROOT/pages/pipes/configuration.adoc | 24 +-
docs/modules/ROOT/pages/pipes/cpu-sizing.adoc | 33 +-
docs/modules/ROOT/pages/pipes/getting-started.adoc | 9 +-
docs/modules/ROOT/pages/pipes/plugins/http.adoc | 47 +-
docs/modules/ROOT/pages/pipes/timeouts.adoc | 485 +++++--
docs/modules/ROOT/pages/pipes/troubleshooting.adoc | 2 +-
docs/modules/ROOT/pages/using-tika/cli/index.adoc | 16 +-
.../ROOT/pages/using-tika/server/index.adoc | 150 +-
llms.txt | 40 +
pom.xml | 2 +
release-tools/uat/run-uat.sh | 51 +-
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 37 +-
.../apache/tika/cli/XmlToJsonConfigConverter.java | 21 +-
.../src/main/java/org/apache/tika/gui/TikaGUI.java | 3 +-
.../test/java/org/apache/tika/cli/TikaCLITest.java | 39 +-
.../tika/cli/XmlToJsonConfigConverterTest.java | 32 +-
.../test/resources/configs/config-template.json | 2 -
.../xml-configs/tika-config-list-map-types.xml | 2 +-
tika-bom/pom.xml | 6 -
tika-bundles/tika-bundle-standard/pom.xml | 2 +-
.../org/apache/tika/config/EmbeddedLimits.java | 4 +-
.../java/org/apache/tika/config/OutputLimits.java | 2 +-
.../java/org/apache/tika/config/ParseTimeout.java | 246 ++++
.../apache/tika/config/TikaProgressTracker.java | 82 --
.../java/org/apache/tika/config/TimeoutLimits.java | 127 +-
.../apache/tika/detect/FileCommandDetector.java | 20 +-
.../tika/detect/MetadataCharsetDetector.java | 15 +-
.../java/org/apache/tika/detect/TypeDetector.java | 3 +-
.../apache/tika/digest/InputStreamDigester.java | 21 +-
.../exception/EmbeddedLimitReachedException.java | 22 +-
.../tika/exception/TikaTimeoutException.java | 71 +-
.../tika/extractor/EmbeddedDocumentUtil.java | 5 +-
.../ParsingEmbeddedDocumentExtractor.java | 45 +-
.../java/org/apache/tika/io/CachingSource.java | 5 +-
.../java/org/apache/tika/io/FilenameUtils.java | 5 +-
.../java/org/apache/tika/io/TikaInputStream.java | 17 +-
.../main/java/org/apache/tika/metadata/Audio.java | 39 +
.../org/apache/tika/metadata/ClimateForcast.java | 55 -
.../org/apache/tika/metadata/ClimateForecast.java | 63 +
.../main/java/org/apache/tika/metadata/DWG.java | 4 +-
.../main/java/org/apache/tika/metadata/Epub.java | 2 +-
.../main/java/org/apache/tika/metadata/HTML.java | 2 +-
.../tika/metadata/{TikaMimeKeys.java => IDML.java} | 16 +-
.../main/java/org/apache/tika/metadata/IPTC.java | 91 +-
.../java/org/apache/tika/metadata/ISO19115.java | 122 ++
.../java/org/apache/tika/metadata/KeyPrefix.java | 182 +++
.../main/java/org/apache/tika/metadata/MAPI.java | 4 +-
.../apache/tika/metadata/{Epub.java => MIF.java} | 24 +-
.../org/apache/tika/metadata/MachineMetadata.java | 8 +-
.../java/org/apache/tika/metadata/Message.java | 2 +-
.../java/org/apache/tika/metadata/Metadata.java | 328 ++++-
.../main/java/org/apache/tika/metadata/Office.java | 2 +-
.../apache/tika/metadata/OfficeOpenXMLCore.java | 7 -
.../tika/metadata/OfficeOpenXMLExtended.java | 2 +-
.../java/org/apache/tika/metadata/OneNote.java | 63 +
.../main/java/org/apache/tika/metadata/PDF.java | 7 +-
.../apache/tika/metadata/PassthroughPrefix.java | 79 --
.../java/org/apache/tika/metadata/Photoshop.java | 2 +-
.../java/org/apache/tika/metadata/Property.java | 227 +++-
.../java/org/apache/tika/metadata/RTFMetadata.java | 2 +-
.../java/org/apache/tika/metadata/Rendering.java | 18 +-
.../apache/tika/metadata/ReservedNamespaces.java | 10 +-
.../main/java/org/apache/tika/metadata/TIFF.java | 4 +-
.../apache/tika/metadata/TikaCoreProperties.java | 156 ++-
.../org/apache/tika/metadata/TikaPagedText.java | 6 +-
.../main/java/org/apache/tika/metadata/XMPDM.java | 84 +-
.../main/java/org/apache/tika/metadata/XMPMM.java | 2 +-
.../main/java/org/apache/tika/metadata/Zip.java | 2 +-
.../metadata/filter/LegacyKeyMigrationFilter.java | 175 ++-
.../filter/RemoveByMimeMetadataFilter.java | 3 +-
.../MetadataWriteLimiter.java | 2 +-
.../MetadataWriteLimiterFactory.java | 4 +-
.../StandardMetadataLimiter.java | 56 +-
.../StandardMetadataLimiterFactory.java | 2 +-
.../main/java/org/apache/tika/mime/MimeTypes.java | 3 +-
.../mime/ProbabilisticMimeDetectionSelector.java | 3 +-
.../org/apache/tika/parser/AutoDetectParser.java | 3 +-
.../org/apache/tika/parser/CompositeParser.java | 19 +-
.../java/org/apache/tika/parser/NetworkParser.java | 40 +-
.../java/org/apache/tika/parser/ParseContext.java | 2 +-
.../java/org/apache/tika/parser/ParseRecord.java | 30 +
.../apache/tika/parser/RecursiveParserWrapper.java | 3 +-
.../tika/parser/external/ExternalParser.java | 13 +-
.../tika/parser/external/ExternalParserConfig.java | 10 +-
.../parser/multiple/AbstractMultipleParser.java | 3 +-
.../sax/AbstractRecursiveParserWrapperHandler.java | 6 -
.../tika/sax/BasicContentHandlerFactory.java | 20 +-
.../org/apache/tika/sax/DIFContentHandler.java | 150 --
.../tika/sax/RecursiveParserWrapperHandler.java | 2 -
.../org/apache/tika/sax/XHTMLContentHandler.java | 3 +-
.../java/org/apache/tika/utils/ExceptionUtils.java | 22 -
.../org/apache/tika/utils/FileProcessResult.java | 39 +
.../java/org/apache/tika/utils/ParserUtils.java | 18 +-
.../java/org/apache/tika/utils/ProcessUtils.java | 167 ++-
.../tika/metadata/metadata-migration-3x-4x.json | 115 +-
.../org/apache/tika/config/ParseTimeoutTest.java | 267 ++++
.../tika/config/TikaProgressTrackerTest.java | 103 --
.../org/apache/tika/config/TimeoutLimitsTest.java | 47 +
.../tika/detect/FileCommandDetectorTest.java | 26 +
.../tika/detect/MetadataCharsetDetectorTest.java | 13 +-
.../org/apache/tika/detect/TypeDetectorTest.java | 3 +-
.../EmbeddedDocumentUtilExtensionTest.java | 9 +-
...arsingEmbeddedDocumentExtractorTimeoutTest.java | 277 ++++
.../java/org/apache/tika/io/FilenameUtilsTest.java | 3 +-
.../org/apache/tika/io/TikaInputStreamTest.java | 11 +-
.../org/apache/tika/metadata/KeyPrefixTest.java | 126 ++
.../metadata/MetadataInternalKeyGuardTest.java | 96 +-
.../tika/metadata/MetadataKeyPrefixRouteTest.java | 130 ++
.../apache/tika/metadata/MetadataPutAllTest.java | 141 ++
.../tika/metadata/MetadataTypedAccessorsTest.java | 117 ++
.../tika/metadata/PropertyReservedNameTest.java | 161 +++
.../org/apache/tika/metadata/PropertyTest.java | 113 ++
.../org/apache/tika/metadata/TestMetadata.java | 61 +-
.../filter/LegacyKeyMigrationFilterTest.java | 131 ++
.../org/apache/tika/mime/MimeDetectionTest.java | 3 +-
.../tika/mime/ProbabilisticMimeDetectionTest.java | 5 +-
.../ProbabilisticMimeDetectionTestWithTika.java | 3 +-
.../apache/tika/parser/CompositeParserTest.java | 61 +-
.../org/apache/tika/parser/NetworkParserTest.java | 135 ++
.../org/apache/tika/parser/mock/MockParser.java | 28 +-
.../tika/parser/multiple/MultipleParserTest.java | 5 +-
.../tika/sax/BasicContentHandlerFactoryTest.java | 33 +
.../org/apache/tika/utils/ProcessUtilsTest.java | 136 ++
.../apache/tika/detect/magika/MagikaDetector.java | 18 +-
.../tika/detect/magika/TestMagikaIntegration.java | 3 +-
.../tika/detect/siegfried/SiegfriedDetector.java | 18 +-
.../resources/tika-config-ignite-handlertype.json | 8 +-
.../test/resources/tika-config-ignite-local.json | 7 +-
.../src/test/resources/tika-config-ignite.json | 7 +-
.../tika-grpc/src/test/resources/tika-config.json | 7 +-
.../apache/tika/server/e2e/RunUatSmokeTest.java | 6 +-
.../tika/server/e2e/TikaServerHttp2Test.java | 6 +-
.../html/StandardHtmlEncodingDetectorTest.java | 5 +-
.../tika/detect/icu4j/Icu4jEncodingDetector.java | 5 +-
.../ml/chardetect/MojibusterEncodingDetector.java | 3 +-
.../universal/UniversalEncodingListener.java | 5 +-
.../org/apache/tika/eval/app/ProfilerBase.java | 16 +-
.../org/apache/tika/eval/app/io/ExtractReader.java | 15 +-
.../apache/tika/eval/app/io/ExtractReaderTest.java | 3 +-
.../src/test/resources/extracts/TIKA-4446-a.json | 2 +-
.../src/test/resources/extracts/TIKA-4446-b.json | 2 +-
.../resources/test-dirs/extractsA/file15_tags.json | 2 +-
.../test-dirs/extractsA/file16_badTags.json | 2 +-
.../test-dirs/extractsA/file17_tagsOutOfOrder.json | 2 +-
.../test/resources/test-dirs/raw_input/file15_tags | 2 +-
.../resources/test-dirs/raw_input/file16_badTags | 2 +-
.../test-dirs/raw_input/file17_tagsOutOfOrder | 2 +-
.../tika/eval/core/util/EvalExceptionUtils.java | 16 +-
.../org/apache/tika/example/ImportContextImpl.java | 3 +-
.../tika/example/MetadataAwareLuceneIndexer.java | 2 +-
.../java/org/apache/tika/example/MyFirstTika.java | 3 +-
.../tika/example/PipesForkParserExample.java | 11 +-
.../org/apache/tika/example/TIAParsingExample.java | 3 +-
.../apache/tika/pipes/grpc/TikaGrpcServerImpl.java | 1 +
.../apache/tika/pipes/grpc/TikaGrpcServerTest.java | 71 +
.../src/test/resources/tika-pipes-test-config.json | 4 +-
tika-metadata-schema/README.md | 29 +-
tika-metadata-schema/pom.xml | 37 +-
tika-metadata-schema/regen.sh | 34 +-
.../tika/metadata/schema/MetadataKeyValidator.java | 81 +-
.../metadata/schema/MigrationTableGenerator.java | 95 +-
.../tika/metadata/schema/SchemaGenerator.java | 140 +-
.../apache/tika/metadata/metadata-key-fields.json | 142 +-
.../org/apache/tika/metadata/metadata-keys.json | 1426 +++++++++++---------
.../tika/metadata/metadata-open-namespaces.json | 61 +-
.../schema/MetadataChangesAdocDriftTest.java | 122 ++
.../tika/metadata/schema/MetadataCoverageTest.java | 16 +-
.../metadata/schema/MetadataKeyValidatorTest.java | 10 +
.../schema/MetadataMigrationTableTest.java | 4 +-
.../metadata/schema/MetadataNoUnderscoreTest.java | 18 +-
.../tika/metadata/schema/MetadataSchemaTest.java | 4 +-
.../metadata/schema/MigrationCompletenessTest.java | 119 ++
.../org/apache/tika/metadata/migration-overlay.tsv | 223 ++-
.../forbidden-apis-metadata-string-key-bans.txt | 32 +
tika-parent/pom.xml | 19 +-
.../java/org/apache/tika/http/TikaHttpClient.java | 185 ++-
.../org/apache/tika/http/TikaHttpClientTest.java | 185 +++
.../org/apache/tika/http/TikaTestHttpServer.java | 38 +-
.../tika/parser/ocrencode/EncodeOCRParser.java | 5 +-
.../tika-parser-scientific-module/pom.xml | 15 +
.../tika/parser/climate/ClimateForecast.java | 64 +
.../apache/tika/parser/envi/EnviHeaderParser.java | 19 +-
.../org/apache/tika/parser/gdal/GDALParser.java | 86 +-
.../geoinfo/GeographicInformationParser.java | 101 +-
.../org/apache/tika/parser/grib/GribParser.java | 33 +-
.../java/org/apache/tika/parser/hdf/HDFParser.java | 17 +-
.../org/apache/tika/parser/isatab/ISATabUtils.java | 14 +-
.../apache/tika/parser/isatab/ISArchiveParser.java | 9 +-
.../apache/tika/parser/netcdf/NetCDFParser.java | 35 +-
.../apache/tika/parser/gdal/TestGDALParser.java | 70 +-
.../geoinfo/GeographicInformationParserTest.java | 12 +-
.../org/apache/tika/parser/hdf/HDFParserTest.java | 8 +-
.../tika/parser/isatab/ISArchiveParserTest.java | 15 +-
.../tika/parser/netcdf/NetCDFParserTest.java | 13 +-
.../tika/inference/AbstractEmbeddingFilter.java | 23 +-
.../tika/inference/ImageEmbeddingConfig.java | 10 +-
.../org/apache/tika/inference/InferenceConfig.java | 12 +-
.../tika/inference/OpenAIEmbeddingFilter.java | 7 +-
.../tika/inference/OpenAIImageEmbeddingParser.java | 26 +-
.../tika/inference/JinaEmbeddingFilterTest.java | 2 +-
.../tika/inference/OpenAIEmbeddingFilterTest.java | 2 +-
.../inference/OpenAIImageEmbeddingParserTest.java | 21 +-
.../tika-parsers-ml/tika-parser-nlp-module/pom.xml | 15 +
.../tika/parser/ctakes/CTAKESContentHandler.java | 10 +-
.../apache/tika/parser/geo/topic/GeoParser.java | 31 +-
.../tika/parser/journal/GrobidRESTParser.java | 21 +-
.../apache/tika/parser/journal/TEIDOMParser.java | 41 +-
.../apache/tika/parser/ner/NamedEntityParser.java | 18 +-
.../tika/parser/geo/topic/GeoParserTest.java | 24 +-
.../tika/parser/journal/JournalParserTest.java | 2 +-
.../org/apache/tika/parser/journal/TEITest.java | 13 +-
.../tika/parser/ner/NamedEntityParserTest.java | 12 +-
.../tika/parser/ner/nltk/NLTKNERecogniserTest.java | 2 +-
.../parser/ner/regex/RegexNERecogniserTest.java | 2 +-
.../tika/parser/ocr/tess4j/Tess4JConfig.java | 19 +-
.../tika/parser/ocr/tess4j/Tess4JParser.java | 155 ++-
.../tika/parser/ocr/tess4j/Tess4JConfigTest.java | 2 +-
.../parser/ocr/tess4j/Tess4JOCRTimeoutTest.java | 250 ++++
.../tika/parser/ocr/tess4j/Tess4JParserTest.java | 6 +-
.../resources/config-examples/tess4j-full.json | 2 +-
.../apache/tika/parser/vlm/AbstractVLMParser.java | 24 +-
.../org/apache/tika/parser/vlm/VLMOCRConfig.java | 12 +-
.../tika/parser/vlm/ClaudeVLMParserTest.java | 13 +-
.../tika/parser/vlm/GeminiVLMParserTest.java | 13 +-
.../tika/parser/vlm/OpenAIVLMParserTest.java | 19 +-
.../resources/config-examples/claude-vlm-full.json | 2 +-
.../resources/config-examples/gemini-vlm-full.json | 2 +-
.../config-examples/openai-vlm-basic.json | 2 +-
.../resources/config-examples/openai-vlm-full.json | 2 +-
.../tika/detect/TestContainerAwareDetector.java | 3 +-
.../java/org/apache/tika/mime/OneOffMimeTest.java | 5 +-
.../apache/tika/parser/AndroidBinaryXMLTest.java | 7 +-
.../tika/parser/AutoDetectParserConfigTest.java | 3 +-
.../apache/tika/parser/AutoDetectParserTest.java | 11 +-
.../tika/parser/AutoDetectReaderParserTest.java | 3 +-
.../tika/parser/RecursiveParserWrapperTest.java | 7 +-
.../apache/tika/parser/apple/PListParserTest.java | 3 +-
.../tika/parser/crypto/Pkcs7EndToEndTest.java | 7 +-
.../apache/tika/parser/crypto/TSDParserTest.java | 3 +-
.../tika/parser/digest/DigestConfigTest.java | 5 +-
.../parser/html/HtmlEncodingDetectionTest.java | 11 +-
.../apache/tika/parser/html/HtmlParserTest.java | 5 +-
.../apache/tika/parser/image/JpegParserTest.java | 5 +-
.../apache/tika/parser/mail/MboxParserTest.java | 3 +-
.../apache/tika/parser/mail/RFC822ParserTest.java | 10 +-
.../tika/parser/microsoft/EMFParserTest.java | 7 +-
.../tika/parser/microsoft/XML2003ParserTest.java | 3 +-
.../microsoft/ooxml/OleObjectRegressionTest.java | 5 +-
.../parser/microsoft/ooxml/TruncatedOOXMLTest.java | 5 +-
.../tika/parser/microsoft/rtf/RTFParserTest.java | 3 +-
.../tika/parser/ocr/TesseractOCRParserTest.java | 21 +-
.../org/apache/tika/parser/odf/ODFParserTest.java | 21 +-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 16 +-
.../org/apache/tika/parser/pkg/ArParserTest.java | 5 +-
.../apache/tika/parser/pkg/Bzip2ParserTest.java | 3 +-
.../apache/tika/parser/pkg/CompressParserTest.java | 3 +-
.../tika/parser/pkg/CompressorParserTest.java | 5 +-
.../org/apache/tika/parser/pkg/GzipParserTest.java | 5 +-
.../org/apache/tika/parser/pkg/RarParserTest.java | 3 +-
.../apache/tika/parser/pkg/Seven7ParserTest.java | 5 +-
.../org/apache/tika/parser/pkg/TarParserTest.java | 3 +-
.../org/apache/tika/parser/pkg/ZipParserTest.java | 3 +-
.../org/apache/tika/parser/pkg/ZlibParserTest.java | 3 +-
.../config-examples/migration-full-example.json | 2 +-
.../resources/config-examples/tesseract-basic.json | 2 +-
.../resources/config-examples/tesseract-full.json | 2 +-
.../tika-parser-apple-module/pom.xml | 15 +
.../org/apache/tika/parser/apple/PListParser.java | 5 +-
.../tika/parser/iwork/IWorkPackageParser.java | 3 +-
.../tika/parser/iwork/KeynoteContentHandler.java | 8 +-
.../tika/parser/iwork/NumbersContentHandler.java | 14 +-
.../tika/parser/iwork/PagesContentHandler.java | 36 +-
.../parser/iwork/iwana/IWork13PackageParser.java | 13 +-
.../parser/iwork/iwana/IWork18PackageParser.java | 3 +-
.../apache/tika/parser/apple/PListParserTest.java | 3 +-
.../apache/tika/parser/iwork/IWorkParserTest.java | 16 +-
.../tika/parser/iwork/iwana/IWork13ParserTest.java | 15 +-
.../tika-parser-audiovideo-module/pom.xml | 15 +
.../org/apache/tika/parser/audio/AudioParser.java | 59 +-
.../org/apache/tika/parser/audio/MidiParser.java | 25 +-
.../org/apache/tika/parser/mp3/ID3v2Frame.java | 16 +-
.../java/org/apache/tika/parser/mp3/Mp3Parser.java | 15 +-
.../java/org/apache/tika/parser/mp4/MP4Parser.java | 44 +-
.../apache/tika/parser/mp4/TikaMp4BoxHandler.java | 7 +-
.../org/apache/tika/parser/mp4/TikaMp4Reader.java | 149 ++
.../tika/parser/mp4/TikaMp4SoundHandler.java | 15 +-
.../tika/parser/mp4/boxes/TikaUserDataBox.java | 174 ++-
.../org/apache/tika/parser/ogg/FlacParser.java | 10 +-
.../org/apache/tika/parser/ogg/OggAudioParser.java | 27 +-
.../java/org/apache/tika/parser/ogg/OggParser.java | 12 +-
.../org/apache/tika/parser/ogg/OpusParser.java | 6 +-
.../org/apache/tika/parser/ogg/SpeexParser.java | 6 +-
.../org/apache/tika/parser/ogg/TheoraParser.java | 5 +-
.../org/apache/tika/parser/ogg/VorbisParser.java | 5 +-
.../org/apache/tika/parser/video/FLVParser.java | 71 +-
.../apache/tika/parser/audio/AudioParserTest.java | 57 +-
.../apache/tika/parser/audio/MidiParserTest.java | 9 +-
.../org/apache/tika/parser/mp3/ID3v2FrameTest.java | 13 +
.../org/apache/tika/parser/mp3/Mp3ParserTest.java | 99 +-
.../org/apache/tika/parser/mp4/MP4ParserTest.java | 150 +-
.../tika/parser/mp4/boxes/TikaUserDataBoxTest.java | 61 +
.../org/apache/tika/parser/ogg/FlacParserTest.java | 9 +-
.../apache/tika/parser/ogg/OggAudioParserTest.java | 3 +-
.../apache/tika/parser/ogg/VorbisParserTest.java | 9 +-
.../apache/tika/parser/video/FLVParserTest.java | 25 +-
.../tika-parser-cad-module/pom.xml | 15 +
.../java/org/apache/tika/parser/dwg/DWGParser.java | 19 +-
.../apache/tika/parser/dwg/DWGParserConfig.java | 10 +-
.../org/apache/tika/parser/dwg/DWGReadParser.java | 13 +-
.../java/org/apache/tika/parser/prt/PRTParser.java | 3 +-
.../org/apache/tika/parser/dwg/DWGParserTest.java | 14 +-
.../org/apache/tika/parser/prt/PRTParserTest.java | 5 +-
.../configs/tika-config-dwgRead-Timeout.json | 2 +-
.../apache/tika/parser/code/SourceCodeParser.java | 7 +-
.../tika/parser/executable/ExecutableParser.java | 37 +-
.../java/org/apache/tika/parser/mat/MatParser.java | 3 +-
.../org/apache/tika/parser/sas/SAS7BDATParser.java | 2 +-
.../tika/parser/code/SourceCodeParserTest.java | 3 +-
.../parser/executable/ExecutableParserTest.java | 9 +-
.../executable/UniversalExecutableParserTest.java | 5 +-
.../apache/tika/parser/sas/SAS7BDATParserTest.java | 4 +-
.../org/apache/tika/parser/crypto/Pkcs7Parser.java | 7 +-
.../org/apache/tika/parser/crypto/TSDParser.java | 3 +-
.../apache/tika/parser/crypto/Pkcs7ParserTest.java | 3 +-
.../tika-parser-font-module/pom.xml | 15 +
.../tika/parser/font/AdobeFontMetricParser.java | 6 +-
.../apache/tika/parser/font/TrueTypeParser.java | 18 +-
.../apache/tika/parser/font/FontParsersTest.java | 5 +-
.../org/apache/tika/parser/html/HtmlHandler.java | 28 +-
.../org/apache/tika/parser/html/JSoupParser.java | 7 +-
.../apache/tika/parser/html/HtmlParserTest.java | 47 +-
.../tika-parser-image-module/pom.xml | 15 +
.../tika/parser/image/AbstractImageParser.java | 9 +-
.../org/apache/tika/parser/image/ICNSParser.java | 12 +-
.../tika/parser/image/ImageMetadataExtractor.java | 82 +-
.../org/apache/tika/parser/image/ImageParser.java | 23 +-
.../org/apache/tika/parser/image/HeicXmpTest.java | 5 +-
.../apache/tika/parser/image/HeifParserTest.java | 7 +-
.../apache/tika/parser/image/ICNSParserTest.java | 5 +-
.../apache/tika/parser/image/ImageParserTest.java | 34 +-
.../apache/tika/parser/image/JpegParserTest.java | 133 +-
.../tika/parser/image/MotionPhotoXmpTest.java | 5 +-
.../apache/tika/parser/image/PSDParserTest.java | 20 +-
.../apache/tika/parser/image/WebPParserTest.java | 5 +-
.../org/apache/tika/parser/image/WebPXmpTest.java | 5 +-
.../apache/tika/parser/jdbc/JDBCTableReader.java | 5 +-
.../tika-parser-mail-module/pom.xml | 15 +
.../tika/parser/mail/MailContentHandler.java | 29 +-
.../org/apache/tika/parser/mbox/MboxParser.java | 34 +-
.../apache/tika/parser/mail/RFC822ParserTest.java | 67 +-
.../apache/tika/parser/mbox/MboxParserTest.java | 28 +-
.../tika-parser-microsoft-module/pom.xml | 15 +
.../microsoft/PSTEmailStreamTranslator.java | 3 +-
.../parser/microsoft/AbstractPOIFSExtractor.java | 15 +-
.../apache/tika/parser/microsoft/EMFParser.java | 3 +-
.../tika/parser/microsoft/JackcessExtractor.java | 9 +-
.../tika/parser/microsoft/JackcessParser.java | 14 +-
.../apache/tika/parser/microsoft/OfficeParser.java | 7 +-
.../tika/parser/microsoft/OutlookExtractor.java | 12 +-
.../tika/parser/microsoft/SummaryExtractor.java | 36 +-
.../apache/tika/parser/microsoft/TNEFParser.java | 3 +-
.../tika/parser/microsoft/chm/ChmParser.java | 3 +-
.../tika/parser/microsoft/libpst/LibPstParser.java | 7 +-
.../microsoft/libpst/LibPstParserConfig.java | 10 +-
.../microsoft/msg/ExtendedMetadataExtractor.java | 15 +-
.../parser/microsoft/onenote/OneNoteParser.java | 44 +-
.../onenote/fsshttpb/MSOneStorePackage.java | 91 +-
.../onenote/fsshttpb/MSOneStoreParser.java | 1 +
.../streamobj/RevisionStoreObjectGroup.java | 3 +
.../microsoft/ooxml/AbstractOOXMLExtractor.java | 5 +-
.../microsoft/ooxml/OOXMLExtractorFactory.java | 5 +-
.../microsoft/ooxml/SAXBasedMetadataExtractor.java | 50 +-
.../parser/microsoft/pst/OutlookPSTParser.java | 7 +-
.../parser/microsoft/pst/PSTMailItemParser.java | 7 +-
.../parser/microsoft/rtf/RTFEmbObjHandler.java | 8 +-
.../parser/microsoft/rtf/RTFObjDataParser.java | 3 +-
.../tika/parser/microsoft/rtf/RTFParser.java | 3 +-
.../microsoft/rtf/jflex/RTFEmbeddedHandler.java | 8 +-
.../rtf/jflex/RTFObjDataStreamParser.java | 3 +-
.../parser/microsoft/xml/SpreadsheetMLParser.java | 3 +-
.../tika/parser/microsoft/xml/WordMLParser.java | 3 +-
.../tika/parser/microsoft/EMFParserTest.java | 3 +-
.../tika/parser/microsoft/ExcelParserTest.java | 82 +-
.../tika/parser/microsoft/OldExcelParserTest.java | 3 +-
.../tika/parser/microsoft/OutlookParserTest.java | 15 +-
.../microsoft/POIContainerExtractionTest.java | 7 +-
.../parser/microsoft/PowerPointParserTest.java | 13 +-
.../tika/parser/microsoft/ProjectParserTest.java | 3 +-
.../tika/parser/microsoft/PublisherParserTest.java | 3 +-
.../parser/microsoft/SolidworksParserTest.java | 13 +-
.../tika/parser/microsoft/VisioParserTest.java | 3 +-
.../tika/parser/microsoft/WMFParserTest.java | 3 +-
.../tika/parser/microsoft/WordParserTest.java | 21 +-
.../microsoft/activemime/ActiveMimeParserTest.java | 3 +-
.../parser/microsoft/libpst/TestLibPstParser.java | 3 +-
.../onenote/fsshttpb/MSOneStorePackageTest.java | 1 -
.../ooxml/OOXMLContainerExtractionTest.java | 3 +-
.../parser/microsoft/ooxml/OOXMLDocxSAXTest.java | 21 +-
.../parser/microsoft/ooxml/OOXMLParserTest.java | 25 +-
.../parser/microsoft/ooxml/OOXMLPptxSAXTest.java | 25 +-
.../parser/microsoft/ooxml/TruncatedOOXMLTest.java | 5 +-
.../parser/microsoft/ooxml/VSDXParserTest.java | 3 +-
.../parser/microsoft/ooxml/xps/XPSParserTest.java | 9 +-
.../parser/microsoft/pst/OutlookPSTParserTest.java | 3 +-
.../tika/parser/microsoft/rtf/RTFParserTest.java | 5 +-
.../parser/microsoft/xml/XML2003ParserTest.java | 5 +-
.../tika-parser-miscoffice-module/pom.xml | 15 +
.../java/org/apache/tika/parser/dbf/DBFParser.java | 5 +-
.../apache/tika/parser/dif/DIFContentHandler.java | 9 +-
.../org/apache/tika/parser/epub/EpubParser.java | 5 +-
.../apache/tika/parser/hwp/HwpTextExtractorV5.java | 3 +-
.../org/apache/tika/parser/hwp/HwpV5Parser.java | 3 +-
.../indesign/ContentAndMetadataExtractor.java | 22 +-
.../apache/tika/parser/indesign/IDMLParser.java | 12 +-
.../apache/tika/parser/mif/MIFContentHandler.java | 7 +-
.../java/org/apache/tika/parser/mif/MIFParser.java | 5 +-
.../tika/parser/odf/FlatOpenDocumentParser.java | 3 +-
.../tika/parser/odf/OpenDocumentMetaParser.java | 17 +-
.../apache/tika/parser/odf/OpenDocumentParser.java | 5 +-
.../tika/parser/wordperfect/QuattroProParser.java | 5 +-
.../tika/parser/wordperfect/WordPerfectParser.java | 13 +-
.../org/apache/tika/parser/dbf/DBFParserTest.java | 11 +-
.../org/apache/tika/parser/dif/DIFParserTest.java | 6 +-
.../apache/tika/parser/epub/EpubParserTest.java | 5 +-
.../apache/tika/parser/hwp/HwpV5ParserTest.java | 5 +-
.../tika/parser/ibooks/iBooksParserTest.java | 4 +-
.../tika/parser/indesign/IDMLParserTest.java | 8 +-
.../org/apache/tika/parser/mif/MIFParserTest.java | 3 +-
.../org/apache/tika/parser/odf/ODFParserTest.java | 19 +-
.../tika/parser/wordperfect/QuattroProTest.java | 8 +-
.../tika/parser/wordperfect/WordPerfectTest.java | 14 +-
.../apache/tika/parser/iptc/IptcAnpaParser.java | 3 +-
.../apache/tika/parser/ocr/TesseractOCRConfig.java | 16 +-
.../apache/tika/parser/ocr/TesseractOCRParser.java | 31 +-
.../tika/parser/ocr/TesseractOCRConfigTest.java | 6 +-
.../tika/parser/ocr/TesseractOCRParserTest.java | 2 +-
.../resources/configs/TIKA-3582-tesseract.json | 2 +-
.../configs/tika-config-tesseract-full.json | 2 +-
.../configs/tika-config-tesseract-partial.json | 2 +-
.../tika-parser-pdf-module/pom.xml | 15 +
.../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 44 +-
.../java/org/apache/tika/parser/pdf/PDFParser.java | 28 +-
.../tika/parser/pdf/PDMetadataExtractor.java | 53 +-
.../org/apache/tika/parser/pdf/XFAExtractor.java | 21 +-
.../tika/parser/pdf/image/ImageGraphicsEngine.java | 16 +-
.../tika/renderer/pdf/pdfbox/PDFBoxRenderer.java | 12 +-
.../tika/renderer/pdf/poppler/PopplerRenderer.java | 18 +-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 127 +-
.../tika/parser/pkg/AbstractArchiveParser.java | 3 +-
.../apache/tika/parser/pkg/CompressorParser.java | 3 +-
.../org/apache/tika/parser/pkg/PackageParser.java | 9 +-
.../java/org/apache/tika/parser/pkg/RarParser.java | 3 +-
.../org/apache/tika/parser/pkg/SevenZParser.java | 3 +-
.../org/apache/tika/parser/pkg/UnrarParser.java | 9 +-
.../java/org/apache/tika/parser/pkg/ZipParser.java | 11 +-
.../apache/tika/parser/pkg/AbstractPkgTest.java | 5 +-
.../tika/parser/pkg/CompressorParserTest.java | 3 +-
.../apache/tika/parser/csv/TextAndCSVParser.java | 19 +-
.../tika/parser/markdown/MarkdownParser.java | 7 +-
.../apache/tika/parser/strings/StringsConfig.java | 24 +-
.../apache/tika/parser/strings/StringsParser.java | 22 +-
.../java/org/apache/tika/parser/txt/TXTParser.java | 7 +-
.../tika/parser/csv/TextAndCSVParserTest.java | 29 +-
.../tika/parser/markdown/MarkdownParserTest.java | 9 +-
.../tika/parser/strings/StringsConfigTest.java | 6 +-
.../org/apache/tika/parser/txt/TXTParserTest.java | 49 +-
.../configs/tika-config-strings-full.json | 2 +-
.../configs/tika-config-strings-partial.json | 2 +-
.../tika-parser-webarchive-module/pom.xml | 15 +
.../org/apache/tika/parser/wacz/WACZParser.java | 3 +-
.../org/apache/tika/parser/warc/WARCParser.java | 21 +-
.../apache/tika/parser/http/HttpParserTest.java | 3 +-
.../apache/tika/parser/wacz/WACZParserTest.java | 5 +-
.../apache/tika/parser/warc/WARCParserTest.java | 11 +-
.../tika-parser-xml-module/pom.xml | 15 +
.../apache/tika/parser/tmx/TMXContentHandler.java | 26 +-
.../java/org/apache/tika/parser/tmx/TMXParser.java | 3 +-
.../tika/parser/xliff/XLIFF12ContentHandler.java | 14 +-
.../apache/tika/parser/xliff/XLIFF12Parser.java | 3 +-
.../tika/parser/xml/AbstractMetadataHandler.java | 33 +-
.../xml/AttributeDependantMetadataHandler.java | 25 +-
.../apache/tika/parser/xml/FictionBookParser.java | 3 +-
.../apache/tika/parser/xml/MetadataHandler.java | 16 +-
.../java/org/apache/tika/parser/xml/XMLParser.java | 5 +-
.../apache/tika/parser/xml/DcXMLParserTest.java | 3 +-
.../parser/xml/TextAndAttributeXMLParserTest.java | 3 +-
.../org/apache/tika/parser/xmp/XmpExtractor.java | 6 +-
.../org/apache/tika/async/cli/PluginsWriter.java | 10 +-
.../apache/tika/async/cli/SimpleAsyncConfig.java | 16 +-
.../org/apache/tika/async/cli/TikaAsyncCLI.java | 9 +-
.../apache/tika/async/cli/AsyncCliParserTest.java | 12 +-
.../apache/tika/async/cli/AsyncProcessorTest.java | 118 ++
.../tika/async/cli/TikaConfigAsyncWriterTest.java | 35 +
.../configs/config-content-only-default.json | 2 -
.../configs/config-content-only-dynamic.json | 2 -
.../test/resources/configs/config-template.json | 2 -
.../org/apache/tika/client/HttpClientFactory.java | 42 +-
.../org/apache/tika/pipes/api/PipesResult.java | 13 +-
.../tika/pipes/core/PerClientServerManager.java | 105 +-
.../org/apache/tika/pipes/core/PipesClient.java | 89 +-
.../org/apache/tika/pipes/core/PipesConfig.java | 153 ++-
.../org/apache/tika/pipes/core/ServerManager.java | 4 +-
.../tika/pipes/core/SharedServerManager.java | 10 +-
.../apache/tika/pipes/core/async/AsyncEmitter.java | 7 +
.../tika/pipes/core/async/AsyncProcessor.java | 20 +-
.../core/extractor/FrictionlessUnpackHandler.java | 3 +-
.../core/extractor/StandardUnpackSelector.java | 3 +-
.../tika/pipes/core/extractor/UnpackExtractor.java | 3 +-
.../tika/pipes/core/protocol/PipesMessage.java | 29 +-
.../serialization/FetchEmitTupleDeserializer.java | 17 +
.../core/serialization/JsonFetchEmitTupleList.java | 24 +-
.../tika/pipes/core/server/ConnectionHandler.java | 87 +-
.../apache/tika/pipes/core/server/EmitHandler.java | 43 +-
.../tika/pipes/core/server/FetchHandler.java | 3 +-
.../tika/pipes/core/server/ParseHandler.java | 11 +-
.../apache/tika/pipes/core/server/PipesServer.java | 156 ++-
.../apache/tika/pipes/core/server/PipesWorker.java | 2 +-
.../tika/pipes/core/server/ServerProtocolIO.java | 29 +
.../pipes/core/server/SharedServerResources.java | 2 +-
.../tika/pipes/core/TikaPipesConfigTest.java | 8 +
.../core/extractor/StandardUnpackSelectorTest.java | 3 +-
.../tika/pipes/core/protocol/PipesMessageTest.java | 4 +-
.../serialization/JsonFetchEmitTupleListTest.java | 28 +
.../UserMetadataReservedKeyStanceTest.java | 60 +
.../WireRestrictedFetchEmitTupleTest.java | 2 +-
.../core/server/EmitHandlerUserMetadataTest.java | 76 ++
.../pipes/core/server/ServerProtocolIOTest.java | 72 +
.../tika/pipes/fork/PipesForkParserTest.java | 5 +-
.../pipes/core/DigestingOpenContainersTest.java | 3 +-
.../apache/tika/pipes/core/EmbeddedLimitsTest.java | 6 +-
.../pipes/core/PartialTimeoutPassbackTest.java | 80 ++
.../apache/tika/pipes/core/PassbackFilterTest.java | 9 +-
.../apache/tika/pipes/core/PipesClientTest.java | 197 ++-
.../pipes/core/async/AsyncChaosMonkeyTest.java | 3 +-
.../configs/tika-config-timeout-lt-heartbeat.json | 4 +-
.../pipes/pipesiterator/PipesIteratorBase.java | 14 +-
.../tika/pipes/emitter/azblob/AZBlobEmitter.java | 5 +-
.../tika-pipes-plugins/tika-pipes-csv/pom.xml | 1 +
.../tika/pipes/iterator/csv/CSVPipesIterator.java | 14 +-
.../pipes/iterator/csv/TestCSVPipesIterator.java | 20 +
.../src/test/resources/test-reserved-header.csv | 2 +
.../tika/pipes/reporter/es/ESReporterConfig.java | 12 +-
.../apache/tika/pipes/es/ConfigExamplesTest.java | 18 +
.../tika/pipes/fetcher/http/HttpFetcher.java | 22 +-
.../fetcher/http/config/HttpFetcherConfig.java | 14 +
.../tika/pipes/fetcher/http/HttpFetcherTest.java | 24 +
.../pipes/iterator/jdbc/JDBCPipesIterator.java | 13 +-
.../pipes/iterator/jdbc/TestJDBCPipesIterator.java | 25 +-
.../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 3 +-
.../tika/config/loader/ComponentInstantiator.java | 5 +
.../tika/serialization/ComponentNameResolver.java | 2 +-
.../tika/serialization/ParseContextUtils.java | 20 +-
.../java/org/apache/tika/config/AllLimitsTest.java | 2 +-
.../org/apache/tika/config/TimeoutLimitsTest.java | 30 +-
.../apache/tika/config/loader/TikaLoaderTest.java | 17 +-
.../tika/metadata/filter/TestMetadataFilter.java | 69 +-
.../StandardMetadataLimiterTest.java | 96 +-
.../WireRestrictedParseContextTest.java | 14 +
.../TIKA-4816-legacy-key-migration-default.json | 5 +
.../configs/TIKA-4816-legacy-key-migration.json | 5 +
tika-server/README.md | 20 +-
.../sample-configs/vision/vlm-openai.json | 2 +-
tika-server/pom.xml | 1 -
tika-server/tika-server-client/pom.xml | 70 -
.../tika/server/client/TikaAsyncHttpClient.java | 37 -
.../org/apache/tika/server/client/TikaClient.java | 69 -
.../apache/tika/server/client/TikaClientCLI.java | 153 ---
.../server/client/TikaClientConfigException.java | 30 -
.../tika/server/client/TikaEmitterResult.java | 54 -
.../tika/server/client/TikaPipesHttpClient.java | 168 ---
.../tika/server/client/TikaServerClientConfig.java | 106 --
.../org/apache/tika/server/client/TestBasic.java | 55 -
.../src/test/resources/log4j2.xml | 32 -
.../resources/tika-config-simple-fs-emitter.json | 11 -
tika-server/tika-server-core/pom.xml | 43 -
.../server/core/BadRequestExceptionMapper.java | 48 +
.../tika/server/core/MaxRequestSizeFilter.java | 138 ++
.../org/apache/tika/server/core/ServerStatus.java | 2 +-
.../org/apache/tika/server/core/TikaServerCli.java | 7 +-
.../apache/tika/server/core/TikaServerConfig.java | 55 +-
.../core/TikaServerParseExceptionMapper.java | 6 +-
.../apache/tika/server/core/TikaServerProcess.java | 167 ++-
.../tika/server/core/resource/AsyncResource.java | 85 +-
.../server/core/resource/DetectorResource.java | 30 +-
.../server/core/resource/LanguageResource.java | 56 +-
.../server/core/resource/MetadataResource.java | 38 +-
.../server/core/resource/PipesParsingHelper.java | 313 +++--
.../tika/server/core/resource/PipesResource.java | 106 +-
.../core/resource/RecursiveMetadataResource.java | 8 +-
.../tika/server/core/resource/TikaResource.java | 401 ++----
.../server/core/resource/TikaResourceAware.java | 18 +-
.../tika/server/core/resource/TikaWelcome.java | 20 +-
.../server/core/resource/TranslateResource.java | 137 --
.../server/core/resource/UnpackerResource.java | 2 -
.../apache/tika/server/core/AsyncResourceTest.java | 242 ++++
.../org/apache/tika/server/core/CXFTestBase.java | 10 +-
.../core/ConfigEndpointSecurityEnabledTest.java | 20 +-
.../server/core/ConfigEndpointSecurityTest.java | 2 +-
.../tika/server/core/LanguageResourceTest.java | 89 +-
.../tika/server/core/MaxRequestSizeFilterTest.java | 127 ++
.../server/core/RecursiveMetadataResourceTest.java | 3 +-
.../apache/tika/server/core/StackTraceTest.java | 87 +-
.../org/apache/tika/server/core/TikaPipesTest.java | 112 +-
.../apache/tika/server/core/TikaResourceTest.java | 155 ++-
.../tika/server/core/TikaServerConfigTest.java | 9 +
.../server/core/TikaServerIntegrationTest.java | 19 +
.../core/TikaServerPipesIntegrationTest.java | 48 +-
.../tika/server/core/TikaServerProcessTest.java | 36 +
.../apache/tika/server/core/TikaWelcomeTest.java | 2 +-
.../tika/server/core/TranslateResourceTest.java | 88 --
.../core/resource/PipesResponseBuilderTest.java | 90 ++
.../core/resource/ReservedComponentIdTest.java | 79 ++
.../test-documents/mock/hello_world_heading.xml} | 23 +-
.../standard/resource/XMPMetadataResource.java | 21 +-
.../tika/server/standard/DetectorResourceTest.java | 10 +-
.../server/standard/JsonMaxFieldLengthTest.java | 2 +-
.../tika/server/standard/MetadataResourceTest.java | 60 +-
.../standard/RecursiveMetadataFilterTest.java | 5 +-
.../standard/RecursiveMetadataResourceTest.java | 25 +-
.../apache/tika/server/standard/TikaPipesTest.java | 4 +-
.../tika/server/standard/TikaResourceTest.java | 48 +-
.../UnpackerResourceConfigDisabledTest.java | 2 +-
.../tika/server/standard/UnpackerResourceTest.java | 2 +-
.../standard/UnpackerResourceWithConfigTest.java | 2 +-
.../resource/ResourceServiceLoadingTest.java | 103 ++
.../main/java/org/apache/tika/xmp/XMPMetadata.java | 6 +-
.../org/apache/tika/xmp/convert/TikaToXMP.java | 3 +-
.../java/org/apache/tika/xmp/TikaToXMPTest.java | 5 +-
655 files changed, 16180 insertions(+), 6567 deletions(-)
diff --cc CHANGES.txt
index e46059d1a1,d2f1ae5bcf..6563f45e4e
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -8,6 -111,6 +111,9 @@@ Release 4.0.0 - ??
OTHER CHANGES
++ * OneNote extraction now follows document order, omits superseded page
++ revisions, and extracts embedded object BLOBs (TIKA-4814).
++
* MagicDetector now compiles its regular expression once, in the
constructor, instead of recompiling it on every match (TIKA-4796).
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
index 8312bdb614,6c1355cb15..8b5a7eabbd
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
@@@ -36,19 -31,13 +35,19 @@@ import java.util.regex.Matcher
import java.util.regex.Pattern;
import org.xml.sax.SAXException;
+import org.xml.sax.helpers.AttributesImpl;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
+import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
- import org.apache.tika.metadata.Property;
+ import org.apache.tika.metadata.OneNote;
import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.microsoft.onenote.OneNotePropertyEnum;
import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.ArrayNumber;
import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.EightBytesOfData;
import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.FourBytesOfData;
import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.IProperty;
@@@ -91,11 -76,6 +90,7 @@@ public class MSOneStorePackage
private static final Pattern HYPERLINK_PATTERN =
Pattern.compile("\uFDDFHYPERLINK\\s+\"([^\"]+)\"([^\"]+)$");
private static final String P = "p";
- /**
- * A visible separator emitted between pages so they can be told apart in plain text
- * output.
- */
- private static final String PAGE_SEPARATOR = "----------------------------------------";
++ private static final int MAX_TRAVERSAL_DEPTH = 100;
static {
LocalDateTime time32Epoch1980 = LocalDateTime.of(1980, Month.JANUARY, 1, 0, 0);
@@@ -197,48 -162,93 +193,41 @@@
public void walkTree(OneNoteTreeWalkerOptions options, Metadata metadata,
XHTMLContentHandler xhtml)
throws SAXException, TikaException, IOException {
- for (RevisionStoreObjectGroup revisionStoreObjectGroup : OtherFileNodeList) {
- for (RevisionStoreObject revisionStoreObject : revisionStoreObjectGroup.objects) {
- PropertySet propertySet =
- revisionStoreObject.propertySet.objectSpaceObjectPropSet.body;
- for (int i = 0; i < propertySet.rgData.size(); ++i) {
- IProperty property = propertySet.rgData.get(i);
- PropertyID propertyID = propertySet.rgPrids[i];
- PropertyType propertyType = PropertyType.fromIntVal(propertyID.type);
- OneNotePropertyEnum oneNotePropertyEnum =
- OneNotePropertyEnum.of(Unsigned.uint(propertyID.value).longValue());
- if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTimeStamp) {
- long fullval = getScalar(property);
- Instant instant = Instant.ofEpochSecond(
- fullval / 10000000 + DATETIME_EPOCH_DIFF_1601);
- if (instant.isAfter(lastModifiedTimestamp)) {
- lastModifiedTimestamp = instant;
- }
- metadata.set(OneNote.LAST_MODIFIED_TIMESTAMP,
- String.valueOf(lastModifiedTimestamp.toEpochMilli()));
- } else if (oneNotePropertyEnum == OneNotePropertyEnum.CreationTimeStamp) {
- // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not
- // 1970
- long scalar = getScalar(property);
- long creationTs = scalar + TIME32_EPOCH_DIFF_1980;
- if (creationTs < creationTimestamp) {
- creationTimestamp = creationTs;
- }
- metadata.set(OneNote.CREATION_TIMESTAMP, String.valueOf(creationTimestamp));
- } else if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTime) {
- // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not
- // 1970
- long scalar = getScalar(property);
- long lastMod = scalar + TIME32_EPOCH_DIFF_1980;
- if (lastMod > lastModified) {
- lastModified = lastMod;
- }
- metadata.set(TikaCoreProperties.MODIFIED, String.valueOf(lastModified));
- } else if (oneNotePropertyEnum == OneNotePropertyEnum.Author) {
- String author =
- new String(((PrtFourBytesOfLengthFollowedByData) property).data,
- StandardCharsets.UTF_8);
- if (mostRecentAuthorProp) {
- mostRecentAuthors.add(author);
- } else if (originalAuthorProp) {
- originalAuthors.add(author);
- } else {
- authors.add(author);
- }
- } else if (oneNotePropertyEnum == OneNotePropertyEnum.AuthorMostRecent) {
- mostRecentAuthorProp = true;
- } else if (oneNotePropertyEnum == OneNotePropertyEnum.AuthorOriginal) {
- originalAuthorProp = true;
- } else if (propertyType == PropertyType.FourBytesOfLengthFollowedByData) {
- boolean isBinary = propertyIsBinary(oneNotePropertyEnum);
- PrtFourBytesOfLengthFollowedByData dataProperty =
- (PrtFourBytesOfLengthFollowedByData) property;
- if ((dataProperty.data.length & 1) == 0 &&
- oneNotePropertyEnum != OneNotePropertyEnum.TextExtendedAscii &&
- !isBinary) {
- if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) {
- xhtml.startElement(P);
- xhtml.characters(
- new String(dataProperty.data, StandardCharsets.UTF_16LE));
- xhtml.endElement(P);
- }
- } else if (oneNotePropertyEnum == OneNotePropertyEnum.TextExtendedAscii) {
- xhtml.startElement(P);
- xhtml.characters(
- new String(dataProperty.data, StandardCharsets.US_ASCII));
- xhtml.endElement(P);
- } else if (!isBinary) {
- if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) {
- xhtml.startElement(P);
- xhtml.characters(
- new String(dataProperty.data, StandardCharsets.UTF_16LE));
- xhtml.endElement(P);
- }
- } else {
- if (oneNotePropertyEnum == OneNotePropertyEnum.RichEditTextUnicode) {
- handleRichEditTextUnicode(dataProperty.data, xhtml);
- } else {
- //TODO -- these seem to be somewhat broken font files and other
- //odds and ends...what are they and how should we process them?
- //handleEmbedded(content.size());
- }
- }
- }
+ walkTree(options, metadata, xhtml, new ParseContext());
+ }
+
+ public void walkTree(OneNoteTreeWalkerOptions options, Metadata metadata,
+ XHTMLContentHandler xhtml, ParseContext parseContext)
+ throws SAXException, TikaException, IOException {
+ this.parseContext = parseContext;
+ this.parentMetadata = metadata;
+ this.embeddedDocumentExtractor =
+ EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(parseContext);
+ if (!cells.isEmpty()) {
+ // Walk each page cell (object space) as a tree, starting from the root objects of
+ // its current revision and following the object references in property order. This
+ // emits the text in document order. The pages are walked in the order in which the
+ // section object space references them; cells that hold older versions of a page
+ // (the same object space in a different revision context) are skipped.
+ List<RevisionStoreCell> pageCells = new ArrayList<>();
+ List<RevisionStoreCell> otherCells = new ArrayList<>();
+ splitCells(pageCells, otherCells);
- boolean firstPage = true;
+ for (RevisionStoreCell cell : pageCells) {
- if (!firstPage) {
- xhtml.startElement(P);
- xhtml.characters(PAGE_SEPARATOR);
- xhtml.endElement(P);
- }
- firstPage = false;
+ xhtml.startElement("div", "class", "page");
+ walkCell(cell, options, metadata, xhtml);
+ xhtml.endElement("div");
+ }
+ for (RevisionStoreCell cell : otherCells) {
+ walkCell(cell, options, metadata, xhtml);
+ }
+ } else {
+ // no cell information available - walk the object groups in revision order
+ Map<ExGuid, RevisionStoreObject> objectsById = indexObjectsById(OtherFileNodeList);
+ Set<ExGuid> visited = new HashSet<>();
+ for (RevisionStoreObjectGroup objectGroup : OtherFileNodeList) {
+ for (RevisionStoreObject object : objectGroup.objects) {
+ walkObject(object, objectsById, visited, AuthorRole.NONE, options, metadata,
- xhtml);
++ xhtml, 0);
}
}
}
@@@ -255,415 -265,6 +244,435 @@@
}
}
+ /**
+ * Splits the cells into page cells, ordered as the section object space references them,
+ * and the remaining cells. A cell that holds an older version of a page - the same object
+ * space referenced by a page cell, but in a different revision context - is dropped, so
+ * content is not emitted once per version snapshot.
+ */
+ private void splitCells(List<RevisionStoreCell> pageCells,
- List<RevisionStoreCell> otherCells) {
++ List<RevisionStoreCell> otherCells) throws TikaException {
+ List<CellID> orderedCellIds = collectSectionReferencedCells();
+ if (orderedCellIds.isEmpty()) {
+ // no page ordering information available - process the cells in storage order
+ pageCells.addAll(cells);
+ return;
+ }
+ Map<CellID, RevisionStoreCell> remainingCells = new LinkedHashMap<>();
+ for (RevisionStoreCell cell : cells) {
+ remainingCells.put(cell.cellID, cell);
+ }
+ Set<ExGuid> coveredObjectSpaces = new HashSet<>();
+ for (CellID cellId : orderedCellIds) {
+ RevisionStoreCell cell = remainingCells.remove(cellId);
+ if (cell != null) {
+ pageCells.add(cell);
+ coveredObjectSpaces.add(cellId.extendGUID2);
+ }
+ }
+ for (RevisionStoreCell cell : remainingCells.values()) {
+ if (cell.cellID == null ||
+ !coveredObjectSpaces.contains(cell.cellID.extendGUID2)) {
+ // not an older version of one of the pages - keep it so no content is lost
+ otherCells.add(cell);
+ }
+ }
+ }
+
+ /**
+ * Walks the section object space (the data root cell) and collects the object space (cell)
+ * references in document order - this is the order of the pages in the section.
+ */
- private List<CellID> collectSectionReferencedCells() {
++ private List<CellID> collectSectionReferencedCells() throws TikaException {
+ List<CellID> orderedCellIds = new ArrayList<>();
+ if (dataRootCell == null) {
+ return orderedCellIds;
+ }
+ Map<ExGuid, RevisionStoreObject> objectsById = indexObjectsById(dataRootCell.objectGroups);
+ Set<ExGuid> visited = new HashSet<>();
+ for (RevisionManifestRootDeclare rootDeclare : dataRootCell.rootDeclares) {
+ collectReferencedCells(objectsById.get(rootDeclare.objectExGuid), objectsById, visited,
- orderedCellIds);
++ orderedCellIds, 0);
+ }
+ for (RevisionStoreObjectGroup objectGroup : dataRootCell.objectGroups) {
+ for (RevisionStoreObject object : objectGroup.objects) {
- collectReferencedCells(object, objectsById, visited, orderedCellIds);
++ collectReferencedCells(object, objectsById, visited, orderedCellIds, 0);
+ }
+ }
+ return orderedCellIds;
+ }
+
+ private void collectReferencedCells(RevisionStoreObject object,
+ Map<ExGuid, RevisionStoreObject> objectsById,
- Set<ExGuid> visited, List<CellID> out) {
++ Set<ExGuid> visited, List<CellID> out, int depth)
++ throws TikaException {
++ if (depth > MAX_TRAVERSAL_DEPTH) {
++ return;
++ }
+ if (object == null || object.propertySet == null ||
+ object.propertySet.objectSpaceObjectPropSet == null) {
+ return;
+ }
+ if (object.objectID != null && !visited.add(object.objectID)) {
+ return;
+ }
+ List<PropertyAction> actions = collectObjectActions(object);
+ for (PropertyAction action : actions) {
+ if (action.spaceReference != null) {
+ out.add(action.spaceReference);
+ } else if (action.isChildReference && action.childReference != null) {
+ collectReferencedCells(objectsById.get(action.childReference), objectsById,
- visited, out);
++ visited, out, depth + 1);
+ }
+ }
+ }
+
+ private void walkCell(RevisionStoreCell cell, OneNoteTreeWalkerOptions options,
+ Metadata metadata, XHTMLContentHandler xhtml)
+ throws SAXException, TikaException, IOException {
+ Map<ExGuid, RevisionStoreObject> objectsById = indexObjectsById(cell.objectGroups);
+ Set<ExGuid> visited = new HashSet<>();
+ // Only objects reachable from the root objects of the current revision are part of
+ // the current content. The object groups may also contain older, superseded versions
+ // of objects (under a different object ID); those are intentionally not walked.
+ for (RevisionManifestRootDeclare rootDeclare : cell.rootDeclares) {
+ walkObject(objectsById.get(rootDeclare.objectExGuid), objectsById, visited,
- AuthorRole.NONE, options, metadata, xhtml);
++ AuthorRole.NONE, options, metadata, xhtml, 0);
+ }
+ if (visited.isEmpty()) {
+ // no root objects could be resolved - walk everything so no content is lost
+ for (RevisionStoreObjectGroup objectGroup : cell.objectGroups) {
+ for (RevisionStoreObject object : objectGroup.objects) {
+ walkObject(object, objectsById, visited, AuthorRole.NONE, options, metadata,
- xhtml);
++ xhtml, 0);
+ }
+ }
+ }
+ }
+
+ /**
+ * Builds a map of object ID to object. The object groups are ordered from the oldest
+ * revision to the newest, so a newer version of an object wins over an older one.
+ */
+ private Map<ExGuid, RevisionStoreObject> indexObjectsById(
+ List<RevisionStoreObjectGroup> objectGroups) {
+ Map<ExGuid, RevisionStoreObject> objectsById = new HashMap<>();
+ for (RevisionStoreObjectGroup objectGroup : objectGroups) {
+ for (RevisionStoreObject object : objectGroup.objects) {
+ if (object.objectID != null) {
+ objectsById.put(object.objectID, object);
+ }
+ }
+ }
+ return objectsById;
+ }
+
+ private void walkObject(RevisionStoreObject object,
+ Map<ExGuid, RevisionStoreObject> objectsById, Set<ExGuid> visited,
+ AuthorRole authorRole, OneNoteTreeWalkerOptions options,
- Metadata metadata, XHTMLContentHandler xhtml)
++ Metadata metadata, XHTMLContentHandler xhtml, int depth)
+ throws SAXException, TikaException, IOException {
++ if (depth > MAX_TRAVERSAL_DEPTH) {
++ throw new TikaException("OneNote object graph exceeds maximum depth of " +
++ MAX_TRAVERSAL_DEPTH);
++ }
+ if (object == null) {
+ return;
+ }
+ if (object.objectID != null && !visited.add(object.objectID)) {
+ return;
+ }
+ if (object.fileDataObject != null) {
+ // the object carries opaque binary data, e.g. an embedded image or file
+ handleEmbedded(object.fileDataObject.getData(), xhtml);
+ }
+ if (object.propertySet == null ||
+ object.propertySet.objectSpaceObjectPropSet == null) {
+ return;
+ }
+ List<PropertyAction> actions = collectObjectActions(object);
+ // An image node can reference the same picture twice: PictureContainer holds the
+ // canonical image data and WebPictureContainer14 holds a rendition derived from it
+ // (e.g. re-rendered when the picture was resized). Only emit the derived rendition
+ // when the canonical container is missing, so the picture is not extracted twice.
+ boolean hasPrimaryPicture = false;
+ for (PropertyAction action : actions) {
+ if (action.isChildReference && action.childReference != null &&
+ action.oneNotePropertyEnum == OneNotePropertyEnum.PictureContainer) {
+ hasPrimaryPicture = true;
+ break;
+ }
+ }
+ // The title structure of a page (StructureElementChildNodes) appears above the page
+ // body on screen, but is declared after the body child nodes. Emit it first so the
+ // text comes out in visual order.
+ for (PropertyAction action : actions) {
+ if (action.oneNotePropertyEnum == OneNotePropertyEnum.StructureElementChildNodes) {
- processAction(action, objectsById, visited, authorRole, options, metadata, xhtml);
++ processAction(action, objectsById, visited, authorRole, options, metadata, xhtml,
++ depth);
+ }
+ }
+ for (PropertyAction action : actions) {
+ if (hasPrimaryPicture &&
+ action.oneNotePropertyEnum == OneNotePropertyEnum.WebPictureContainer14) {
+ continue;
+ }
+ if (action.oneNotePropertyEnum != OneNotePropertyEnum.StructureElementChildNodes) {
- processAction(action, objectsById, visited, authorRole, options, metadata, xhtml);
++ processAction(action, objectsById, visited, authorRole, options, metadata, xhtml,
++ depth);
+ }
+ }
+ }
+
+ /**
+ * A property of an object, together with the object reference assigned to it if it is
+ * an object reference property.
+ */
+ private static final class PropertyAction {
+ private final IProperty property;
+ private final PropertyType propertyType;
+ private final OneNotePropertyEnum oneNotePropertyEnum;
+ private final boolean isChildReference;
+ private final ExGuid childReference;
+ private final CellID spaceReference;
+
+ PropertyAction(IProperty property, PropertyType propertyType,
+ OneNotePropertyEnum oneNotePropertyEnum, boolean isChildReference,
+ ExGuid childReference, CellID spaceReference) {
+ this.property = property;
+ this.propertyType = propertyType;
+ this.oneNotePropertyEnum = oneNotePropertyEnum;
+ this.isChildReference = isChildReference;
+ this.childReference = childReference;
+ this.spaceReference = spaceReference;
+ }
+ }
+
+ /**
+ * Flattens the properties of an object, in order, into a list of actions.
+ */
- private List<PropertyAction> collectObjectActions(RevisionStoreObject object) {
++ private List<PropertyAction> collectObjectActions(RevisionStoreObject object)
++ throws TikaException {
+ List<ExGuid> referencedObjects =
+ object.referencedObjectID == null || object.referencedObjectID.content == null ?
+ Collections.emptyList() : object.referencedObjectID.content;
+ List<CellID> referencedSpaces = object.referencedObjectSpacesID == null ||
+ object.referencedObjectSpacesID.content == null ? Collections.emptyList() :
+ object.referencedObjectSpacesID.content;
+ List<PropertyAction> actions = new ArrayList<>();
+ collectActions(object.propertySet.objectSpaceObjectPropSet.body, referencedObjects,
- new int[]{0}, referencedSpaces, new int[]{0}, actions);
++ new int[]{0}, referencedSpaces, new int[]{0}, actions, 0);
+ return actions;
+ }
+
+ /**
+ * Flattens the properties of a property set, in order, into a list of actions.
+ * Properties of type ObjectID or ArrayOfObjectIDs consume, in property order, the object
+ * references of the containing object, and properties of type ObjectSpaceID or
+ * ArrayOfObjectSpaceIDs consume the object space (cell) references (see MS-ONESTORE
+ * section 2.7.8), so the references must be assigned here, in property order, no matter
+ * in which order the actions are processed later.
+ */
+ private void collectActions(PropertySet propertySet, List<ExGuid> referencedObjects,
+ int[] referenceCursor, List<CellID> referencedSpaces,
- int[] spaceCursor, List<PropertyAction> actions) {
++ int[] spaceCursor, List<PropertyAction> actions, int depth)
++ throws TikaException {
++ if (depth > MAX_TRAVERSAL_DEPTH) {
++ throw new TikaException("OneNote property graph exceeds maximum depth of " +
++ MAX_TRAVERSAL_DEPTH);
++ }
+ if (propertySet == null || propertySet.rgPrids == null || propertySet.rgData == null) {
+ return;
+ }
+ for (int i = 0; i < propertySet.rgPrids.length && i < propertySet.rgData.size(); ++i) {
+ IProperty property = propertySet.rgData.get(i);
+ PropertyID propertyID = propertySet.rgPrids[i];
+ PropertyType propertyType = PropertyType.fromIntVal(propertyID.type);
+ OneNotePropertyEnum oneNotePropertyEnum =
+ OneNotePropertyEnum.of(Unsigned.uint(propertyID.value).longValue());
+ if (propertyType == PropertyType.ObjectID) {
+ actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, true,
+ nextReference(referencedObjects, referenceCursor), null));
+ } else if (propertyType == PropertyType.ArrayOfObjectIDs) {
- int count = property instanceof ArrayNumber ? ((ArrayNumber) property).number : 0;
++ int count = property instanceof ArrayNumber ?
++ Math.min(Math.max(0, ((ArrayNumber) property).number),
++ referencedObjects.size() - referenceCursor[0]) : 0;
+ for (int j = 0; j < count; ++j) {
+ actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum,
+ true, nextReference(referencedObjects, referenceCursor), null));
+ }
+ } else if (propertyType == PropertyType.ObjectSpaceID) {
+ actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, false,
+ null, nextSpaceReference(referencedSpaces, spaceCursor)));
+ } else if (propertyType == PropertyType.ArrayOfObjectSpaceIDs) {
- int count = property instanceof ArrayNumber ? ((ArrayNumber) property).number : 0;
++ int count = property instanceof ArrayNumber ?
++ Math.min(Math.max(0, ((ArrayNumber) property).number),
++ referencedSpaces.size() - spaceCursor[0]) : 0;
+ for (int j = 0; j < count; ++j) {
+ actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum,
+ false, null, nextSpaceReference(referencedSpaces, spaceCursor)));
+ }
+ } else if (propertyType == PropertyType.PropertySet) {
+ if (property instanceof PropertySet) {
+ collectActions((PropertySet) property, referencedObjects, referenceCursor,
- referencedSpaces, spaceCursor, actions);
++ referencedSpaces, spaceCursor, actions, depth + 1);
+ }
+ } else if (propertyType == PropertyType.ArrayOfPropertyValues) {
+ if (property instanceof PrtArrayOfPropertyValues &&
+ ((PrtArrayOfPropertyValues) property).data != null) {
+ for (PropertySet nested : ((PrtArrayOfPropertyValues) property).data) {
+ collectActions(nested, referencedObjects, referenceCursor,
- referencedSpaces, spaceCursor, actions);
++ referencedSpaces, spaceCursor, actions, depth + 1);
+ }
+ }
+ } else {
+ actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, false,
+ null, null));
+ }
+ }
+ }
+
+ private void processAction(PropertyAction action,
+ Map<ExGuid, RevisionStoreObject> objectsById, Set<ExGuid> visited,
+ AuthorRole authorRole, OneNoteTreeWalkerOptions options,
- Metadata metadata, XHTMLContentHandler xhtml)
++ Metadata metadata, XHTMLContentHandler xhtml, int depth)
+ throws SAXException, TikaException, IOException {
+ if (action.spaceReference != null) {
+ // a reference to another object space (cell) - cells are walked separately
+ return;
+ }
+ if (action.isChildReference) {
+ AuthorRole childRole = AuthorRole.NONE;
+ if (action.oneNotePropertyEnum == OneNotePropertyEnum.AuthorMostRecent) {
+ childRole = AuthorRole.MOST_RECENT;
+ } else if (action.oneNotePropertyEnum == OneNotePropertyEnum.AuthorOriginal) {
+ childRole = AuthorRole.ORIGINAL;
+ }
+ walkObject(action.childReference == null ? null :
+ objectsById.get(action.childReference), objectsById, visited,
- childRole, options, metadata, xhtml);
++ childRole, options, metadata, xhtml, depth + 1);
+ } else {
+ processPrimitiveProperty(action.property, action.propertyType,
+ action.oneNotePropertyEnum, authorRole, options, metadata, xhtml);
+ }
+ }
+
+ private ExGuid nextReference(List<ExGuid> referencedObjects, int[] referenceCursor) {
+ if (referenceCursor[0] < referencedObjects.size()) {
+ return referencedObjects.get(referenceCursor[0]++);
+ }
+ return null;
+ }
+
+ private CellID nextSpaceReference(List<CellID> referencedSpaces, int[] spaceCursor) {
+ if (spaceCursor[0] < referencedSpaces.size()) {
+ return referencedSpaces.get(spaceCursor[0]++);
+ }
+ return null;
+ }
+
+ private void processPrimitiveProperty(IProperty property, PropertyType propertyType,
+ OneNotePropertyEnum oneNotePropertyEnum,
+ AuthorRole authorRole,
+ OneNoteTreeWalkerOptions options, Metadata metadata,
+ XHTMLContentHandler xhtml)
+ throws SAXException, TikaException, IOException {
+ if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTimeStamp) {
+ long fullval = getScalar(property);
+ Instant instant = Instant.ofEpochSecond(
+ fullval / 10000000 + DATETIME_EPOCH_DIFF_1601);
+ if (instant.isAfter(lastModifiedTimestamp)) {
+ lastModifiedTimestamp = instant;
+ }
- metadata.set(ONE_NOTE_PREFIX + "lastModifiedTimestamp",
++ metadata.set(OneNote.LAST_MODIFIED_TIMESTAMP,
+ String.valueOf(lastModifiedTimestamp.toEpochMilli()));
+ } else if (oneNotePropertyEnum == OneNotePropertyEnum.CreationTimeStamp) {
+ // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not
+ // 1970
+ long scalar = getScalar(property);
+ long creationTs = scalar + TIME32_EPOCH_DIFF_1980;
+ if (creationTs < creationTimestamp) {
+ creationTimestamp = creationTs;
+ }
- metadata.set(ONE_NOTE_PREFIX + "creationTimestamp", String.valueOf(creationTimestamp));
++ metadata.set(OneNote.CREATION_TIMESTAMP, String.valueOf(creationTimestamp));
+ } else if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTime) {
+ // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not
+ // 1970
+ long scalar = getScalar(property);
+ long lastMod = scalar + TIME32_EPOCH_DIFF_1980;
+ if (lastMod > lastModified) {
+ lastModified = lastMod;
+ }
+ metadata.set(TikaCoreProperties.MODIFIED, String.valueOf(lastModified));
+ } else if (oneNotePropertyEnum == OneNotePropertyEnum.Author) {
+ String author = new String(((PrtFourBytesOfLengthFollowedByData) property).data,
+ StandardCharsets.UTF_8);
+ if (authorRole == AuthorRole.MOST_RECENT) {
+ mostRecentAuthors.add(author);
+ } else if (authorRole == AuthorRole.ORIGINAL) {
+ originalAuthors.add(author);
+ // the original authors are the creators of the content
+ authors.add(author);
+ } else {
+ authors.add(author);
+ }
+ } else if (propertyType == PropertyType.FourBytesOfLengthFollowedByData) {
+ boolean isBinary = propertyIsBinary(oneNotePropertyEnum);
+ PrtFourBytesOfLengthFollowedByData dataProperty =
+ (PrtFourBytesOfLengthFollowedByData) property;
+ if ((dataProperty.data.length & 1) == 0 &&
+ oneNotePropertyEnum != OneNotePropertyEnum.TextExtendedAscii && !isBinary) {
+ if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) {
+ xhtml.startElement(P);
+ xhtml.characters(new String(dataProperty.data, StandardCharsets.UTF_16LE));
+ xhtml.endElement(P);
+ }
+ } else if (oneNotePropertyEnum == OneNotePropertyEnum.TextExtendedAscii) {
+ xhtml.startElement(P);
+ xhtml.characters(new String(dataProperty.data, StandardCharsets.US_ASCII));
+ xhtml.endElement(P);
+ } else if (!isBinary) {
+ if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) {
+ xhtml.startElement(P);
+ xhtml.characters(new String(dataProperty.data, StandardCharsets.UTF_16LE));
+ xhtml.endElement(P);
+ }
+ } else {
+ if (oneNotePropertyEnum == OneNotePropertyEnum.RichEditTextUnicode) {
+ handleRichEditTextUnicode(dataProperty.data, xhtml);
+ } else {
+ //TODO -- these seem to be somewhat broken font files and other
+ //odds and ends...what are they and how should we process them?
+ //handleEmbedded(content.size());
+ }
+ }
+ }
+ }
+
+
+ /**
+ * Hands the binary data of an embedded object (e.g. an image or an attached file) to the
+ * embedded document extractor.
+ */
+ private void handleEmbedded(byte[] data, XHTMLContentHandler xhtml)
+ throws SAXException, IOException {
+ if (data == null || data.length == 0 || embeddedDocumentExtractor == null) {
+ return;
+ }
+ Metadata embeddedMetadata = Metadata.newInstance(this.parseContext);
+ AttributesImpl attributes = new AttributesImpl();
+ attributes.addAttribute("", "class", "class", "CDATA", "embedded");
+ xhtml.startElement("div", attributes);
+ xhtml.endElement("div");
+ try (TikaInputStream tis = TikaInputStream.get(data)) {
+ if (embeddedDocumentExtractor.shouldParseEmbedded(embeddedMetadata)) {
+ embeddedDocumentExtractor.parseEmbedded(tis, new EmbeddedContentHandler(xhtml),
+ embeddedMetadata, this.parseContext, false);
+ }
+ } catch (IOException e) {
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata);
+ }
+ }
private void handleRichEditTextUnicode(byte[] arr, XHTMLContentHandler xhtml)
throws SAXException, IOException, TikaException {
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java
index a44a9f75ed,25790ead8b..30ce905c67
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java
@@@ -69,10 -55,8 +69,11 @@@ public class MSOneStoreParser
private List<DataElement> objectGroupDataElements;
// The DataElements of Object BLOB
private List<DataElement> objectBlOBElements;
+ // The DataElements of Object BLOB, keyed by their data element extended GUID
+ private Map<ExGuid, DataElement> objectBlOBElementsById;
public MSOneStorePackage parse(DataElementPackage dataElementPackage) throws IOException {
++ storageIndexHashTab.clear();
MSOneStorePackage msOneStorePackage = new MSOneStorePackage();
storageIndexDataElements = dataElementPackage.dataElements.stream()
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java
index 6de0064661,afd2f6c067..dfa3f05d8f
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java
@@@ -83,6 -73,6 +83,9 @@@ public class RevisionStoreObjectGroup
for (int i = 0; i <
dataObject.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList.size();
i++) {
++ if (i >= dataObject.objectGroupData.objectGroupObjectDataBLOBReferenceList.size()) {
++ throw new IOException("Missing BLOB reference for object declaration " + i);
++ }
ObjectGroupObjectBLOBDataDeclaration objectGroupObjectBLOBDataDeclaration =
dataObject.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList.get(
i);
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
index 841415bb98,0000000000..e164455dab
mode 100644,000000..100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
@@@ -1,293 -1,0 +1,292 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.onenote.fsshttpb;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.ArrayNumber;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.IProperty;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.NoData;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtArrayOfPropertyValues;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtFourBytesOfLengthFollowedByData;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectData;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySet;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySetObject;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestRootDeclare;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreCell;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObject;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObjectGroup;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellID;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGuid;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyID;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyType;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectPropSet;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil;
+import org.apache.tika.sax.ToTextContentHandler;
+import org.apache.tika.sax.XHTMLContentHandler;
+
+public class MSOneStorePackageTest {
+
+ @Test
+ public void testPagesFollowSectionOrderAndDropOlderCellVersions() throws Exception {
+ ExGuid sectionRootId = id(1);
+ CellID pageOne = cell(10, 100);
+ CellID pageTwo = cell(20, 200);
+ CellID oldPageOne = cell(11, 100);
+ RevisionStoreCell pageTwoCell = cellWithText(pageTwo, "page two");
+ RevisionStoreCell oldPageOneCell = cellWithText(oldPageOne, "old page one");
+ RevisionStoreCell pageOneCell = cellWithText(pageOne, "page one");
+ RevisionStoreCell unrelatedCell = cellWithText(cell(30, 300), "unrelated");
+
+ RevisionStoreObject sectionRoot = object(sectionRootId,
+ propertySet(new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78,
+ new NoData()),
+ new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D79,
+ new NoData())),
+ Collections.emptyList(), Arrays.asList(pageOne, pageTwo));
+ RevisionStoreCell section = new RevisionStoreCell();
+ section.objectGroups.add(group(sectionRoot));
+ RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare();
+ rootDeclare.objectExGuid = sectionRootId;
+ section.rootDeclares.add(rootDeclare);
+ RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare();
+ missingRoot.objectExGuid = id(1000);
+ section.rootDeclares.add(missingRoot);
+
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.dataRootCell = section;
+ pkg.cells.addAll(Arrays.asList(pageTwoCell, oldPageOneCell, pageOneCell, unrelatedCell));
+
+ String text = walk(pkg);
+ assertTrue(text.indexOf("page one") < text.indexOf("page two"));
- assertTrue(text.contains("----------------------------------------"));
+ assertFalse(text.contains("old page one"));
+ assertTrue(text.contains("unrelated"));
+ }
+
+ @Test
+ public void testUnresolvedRootsFallBackToAllObjects() throws Exception {
+ RevisionStoreCell cell = cellWithText(cell(1, 1), "fallback content");
+ cell.rootDeclares.clear();
+ RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare();
+ missingRoot.objectExGuid = id(999);
+ cell.rootDeclares.add(missingRoot);
+
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ assertTrue(walk(pkg).contains("fallback content"));
+ }
+
+ @Test
+ public void testPrimaryPictureSuppressesDerivedPicture() throws Exception {
+ ExGuid pictureID = id(40);
+ ExGuid webPictureID = id(41);
+ RevisionStoreObject root = object(id(42), propertySet(
+ new PropertySpec(PropertyType.ObjectID, 0x20001C3F, new NoData()),
+ new PropertySpec(PropertyType.ObjectID, 0x200034C8, new NoData())),
+ Arrays.asList(pictureID, webPictureID), Collections.emptyList());
+ RevisionStoreObject picture = object(pictureID, propertySet(),
+ Collections.emptyList(), Collections.emptyList());
+ picture.propertySet.objectSpaceObjectPropSet.body = null;
+ RevisionStoreObject webPicture = object(webPictureID, propertySet(
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("derived picture"))),
+ Collections.emptyList(), Collections.emptyList());
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(root, picture, webPicture));
+ RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare();
+ rootDeclare.objectExGuid = root.objectID;
+ cell.rootDeclares.add(rootDeclare);
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ assertFalse(walk(pkg).contains("derived picture"));
+ }
+
+ @Test
+ public void testNestedPropertySetsAndMissingReferencesAreTraversedSafely() throws Exception {
+ ExGuid childId = id(2);
+ PropertySet nested = propertySet(new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("nested text")));
+ PrtArrayOfPropertyValues array = new PrtArrayOfPropertyValues();
+ array.data = new PropertySet[]{propertySet(new PropertySpec(
+ PropertyType.FourBytesOfLengthFollowedByData, 0x1C003498, text("array text")))};
+ RevisionStoreObject root = object(id(1), propertySet(
+ new PropertySpec(PropertyType.PropertySet, 0, nested),
+ new PropertySpec(PropertyType.ArrayOfPropertyValues, 0, array),
+ new PropertySpec(PropertyType.ArrayOfObjectIDs, 0x24001D5F,
+ arrayNumber(2)),
+ new PropertySpec(PropertyType.ObjectID, 0x24001D5F, new NoData()),
+ new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78, new NoData()),
+ new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D79, new NoData()),
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001DD7, bytes((byte) 'u', (byte) 0, (byte) 1)),
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001C22, bytes((byte) 'h', (byte) 0, (byte) 'i', (byte) 0,
+ (byte) 0, (byte) 0))),
+ Collections.singletonList(childId), Collections.singletonList(cell(50, 51)));
+ RevisionStoreObject child = object(childId, propertySet(
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("child text"))),
+ Collections.emptyList(), Collections.emptyList());
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(root, child));
+ RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare();
+ rootDeclare.objectExGuid = root.objectID;
+ cell.rootDeclares.add(rootDeclare);
+
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+ String text = walk(pkg);
+ assertTrue(text.contains("nested text"));
+ assertTrue(text.contains("array text"));
+ assertTrue(text.contains("child text"));
+ assertTrue(text.contains("u"));
+ assertTrue(text.contains("hi"));
+ }
+
+ private static String walk(MSOneStorePackage pkg) throws Exception {
+ Metadata metadata = new Metadata();
+ StringWriter writer = new StringWriter();
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(
+ new ToTextContentHandler(writer), metadata, new ParseContext());
+ xhtml.startDocument();
+ pkg.walkTree(new OneNoteTreeWalkerOptions(), metadata, xhtml);
+ xhtml.endDocument();
+ return writer.toString();
+ }
+
+ private static RevisionStoreCell cellWithText(CellID cellID, String value) throws Exception {
+ RevisionStoreObject object = object(id(cellID.extendGUID1.hashCode()),
+ propertySet(new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text(value))), Collections.emptyList(), Collections.emptyList());
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.cellID = cellID;
+ cell.objectGroups.add(group(object));
+ RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare();
+ rootDeclare.objectExGuid = object.objectID;
+ cell.rootDeclares.add(rootDeclare);
+ return cell;
+ }
+
+ private static RevisionStoreObject object(ExGuid objectID, PropertySet body,
+ List<ExGuid> references, List<CellID> spaces)
+ throws Exception {
+ RevisionStoreObject object = new RevisionStoreObject();
+ object.objectID = objectID;
+ PropertySetObject propertySetObject = new PropertySetObject(null, emptyObjectData());
+ ObjectSpaceObjectPropSet propSet = new ObjectSpaceObjectPropSet();
+ propSet.body = body;
+ propertySetObject.objectSpaceObjectPropSet = propSet;
+ object.propertySet = propertySetObject;
+ if (!references.isEmpty()) {
+ object.referencedObjectID = new org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGUIDArray();
+ object.referencedObjectID.content = references;
+ }
+ if (!spaces.isEmpty()) {
+ object.referencedObjectSpacesID = new org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellIDArray();
+ object.referencedObjectSpacesID.content = spaces;
+ }
+ return object;
+ }
+
+ private static RevisionStoreObjectGroup group(RevisionStoreObject... objects) {
+ RevisionStoreObjectGroup group = new RevisionStoreObjectGroup(id(500));
+ group.objects.addAll(Arrays.asList(objects));
+ return group;
+ }
+
+ private static PropertySet propertySet(PropertySpec... specs) {
+ PropertySet set = new PropertySet();
+ set.cProperties = specs.length;
+ set.rgPrids = new PropertyID[specs.length];
+ set.rgData = new ArrayList<>();
+ for (int i = 0; i < specs.length; i++) {
+ set.rgPrids[i] = propertyID(specs[i].type, specs[i].value);
+ set.rgData.add(specs[i].property);
+ }
+ return set;
+ }
+
+ private static PropertyID propertyID(PropertyType type, int value) {
+ PropertyID id = new PropertyID();
+ id.type = type.getIntVal();
+ id.value = value;
+ return id;
+ }
+
+ private static PrtFourBytesOfLengthFollowedByData bytes(byte... value) {
+ PrtFourBytesOfLengthFollowedByData data = new PrtFourBytesOfLengthFollowedByData();
+ data.data = value;
+ data.cb = data.data.length;
+ return data;
+ }
+
+ private static PrtFourBytesOfLengthFollowedByData text(String value) {
+ PrtFourBytesOfLengthFollowedByData data = new PrtFourBytesOfLengthFollowedByData();
+ data.data = value.getBytes(StandardCharsets.US_ASCII);
+ data.cb = data.data.length;
+ return data;
+ }
+
+ private static ArrayNumber arrayNumber(int number) {
+ ArrayNumber array = new ArrayNumber();
+ array.number = number;
+ return array;
+ }
+
+ private static ObjectGroupObjectData emptyObjectData() {
+ ObjectGroupObjectData data = new ObjectGroupObjectData();
+ data.data.content.addAll(ByteUtil.toListOfByte(new byte[]{0, 0, 0, (byte) 0x80,
+ 0, 0, 0, 0}));
+ return data;
+ }
+
+ private static CellID cell(int first, int second) {
+ return new CellID(id(first), id(second));
+ }
+
+ private static ExGuid id(int value) {
+ return new ExGuid(value, UUID.nameUUIDFromBytes(("id-" + value).getBytes(StandardCharsets.UTF_8)));
+ }
+
+ private static final class PropertySpec {
+ private final PropertyType type;
+ private final int value;
+ private final IProperty property;
+
+ private PropertySpec(PropertyType type, int value, IProperty property) {
+ this.type = type;
+ this.value = value;
+ this.property = property;
+ }
+ }
+}