You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by nd...@apache.org on 2026/08/13 22:38:59 UTC
(tika) 01/01: Merge remote-tracking branch 'origin/main' into pr-3018
This is an automated email from the ASF dual-hosted git repository.
nddipiazza pushed a commit to branch pr-3018
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 6f07d05d0fd8fcba9df8fc844283f008ee45d568
Merge: 177d21b9da 7d6fe60962
Author: nddipiazza <nd...@users.noreply.github.com>
AuthorDate: Thu Aug 13 17:38:30 2026 -0500
Merge remote-tracking branch 'origin/main' into pr-3018
.devcontainer/Dockerfile | 22 +
.../index.adoc => .devcontainer/devcontainer.json | 26 +-
.github/workflows/main-jdk17-build.yml | 5 +-
.skills/dev.md | 133 --
.skills/dev/SKILL.md | 210 +++
.../SKILL.md} | 34 +-
.skills/pr-review/SKILL.md | 211 +++
.../SKILL.md} | 31 +-
.../SKILL.md} | 25 +
.../SKILL.md} | 25 +
AGENTS.md | 34 +
CHANGES.txt | 117 ++
.../integration-testing/run-uat-script.adoc | 14 +-
.../advanced/integration-testing/tika-app.adoc | 2 +-
.../integration-testing/tika-eval-regression.adoc | 3 +-
.../advanced/integration-testing/tika-server.adoc | 26 +-
.../ROOT/pages/advanced/local-vlm-server.adoc | 6 +-
.../ROOT/pages/advanced/setting-limits.adoc | 68 +-
.../configuration/parsers/external-parser.adoc | 2 +-
.../pages/configuration/parsers/tess4j-parser.adoc | 9 +-
.../pages/configuration/parsers/vlm-parsers.adoc | 6 +-
docs/modules/ROOT/pages/developers/index.adoc | 2 +-
.../ROOT/pages/developers/metadata-keys.adoc | 43 +-
.../ROOT/pages/developers/serialization.adoc | 19 +-
.../pages/migration-to-4x/metadata-changes-4x.adoc | 226 +++-
.../migration-to-4x/migrating-tika-server-4x.adoc | 229 +++-
.../pages/migration-to-4x/migrating-to-4x.adoc | 44 +-
docs/modules/ROOT/pages/pipes/configuration.adoc | 24 +-
docs/modules/ROOT/pages/pipes/cpu-sizing.adoc | 33 +-
docs/modules/ROOT/pages/pipes/getting-started.adoc | 9 +-
docs/modules/ROOT/pages/pipes/plugins/http.adoc | 47 +-
docs/modules/ROOT/pages/pipes/timeouts.adoc | 485 +++++--
docs/modules/ROOT/pages/pipes/troubleshooting.adoc | 2 +-
docs/modules/ROOT/pages/using-tika/cli/index.adoc | 16 +-
.../ROOT/pages/using-tika/server/index.adoc | 150 +-
llms.txt | 40 +
pom.xml | 2 +
release-tools/uat/run-uat.sh | 51 +-
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 37 +-
.../apache/tika/cli/XmlToJsonConfigConverter.java | 21 +-
.../src/main/java/org/apache/tika/gui/TikaGUI.java | 3 +-
.../test/java/org/apache/tika/cli/TikaCLITest.java | 39 +-
.../tika/cli/XmlToJsonConfigConverterTest.java | 32 +-
.../test/resources/configs/config-template.json | 2 -
.../xml-configs/tika-config-list-map-types.xml | 2 +-
tika-bom/pom.xml | 6 -
tika-bundles/tika-bundle-standard/pom.xml | 2 +-
.../org/apache/tika/config/EmbeddedLimits.java | 4 +-
.../java/org/apache/tika/config/OutputLimits.java | 2 +-
.../java/org/apache/tika/config/ParseTimeout.java | 246 ++++
.../apache/tika/config/TikaProgressTracker.java | 82 --
.../java/org/apache/tika/config/TimeoutLimits.java | 127 +-
.../apache/tika/detect/FileCommandDetector.java | 20 +-
.../tika/detect/MetadataCharsetDetector.java | 15 +-
.../java/org/apache/tika/detect/TypeDetector.java | 3 +-
.../apache/tika/digest/InputStreamDigester.java | 21 +-
.../exception/EmbeddedLimitReachedException.java | 22 +-
.../tika/exception/TikaTimeoutException.java | 71 +-
.../tika/extractor/EmbeddedDocumentUtil.java | 5 +-
.../ParsingEmbeddedDocumentExtractor.java | 45 +-
.../java/org/apache/tika/io/CachingSource.java | 5 +-
.../java/org/apache/tika/io/FilenameUtils.java | 5 +-
.../java/org/apache/tika/io/TikaInputStream.java | 17 +-
.../main/java/org/apache/tika/metadata/Audio.java | 39 +
.../org/apache/tika/metadata/ClimateForcast.java | 55 -
.../org/apache/tika/metadata/ClimateForecast.java | 63 +
.../main/java/org/apache/tika/metadata/DWG.java | 4 +-
.../main/java/org/apache/tika/metadata/Epub.java | 2 +-
.../main/java/org/apache/tika/metadata/HTML.java | 2 +-
.../tika/metadata/{TikaMimeKeys.java => IDML.java} | 16 +-
.../main/java/org/apache/tika/metadata/IPTC.java | 91 +-
.../java/org/apache/tika/metadata/ISO19115.java | 122 ++
.../java/org/apache/tika/metadata/KeyPrefix.java | 182 +++
.../main/java/org/apache/tika/metadata/MAPI.java | 4 +-
.../apache/tika/metadata/{Epub.java => MIF.java} | 24 +-
.../org/apache/tika/metadata/MachineMetadata.java | 8 +-
.../java/org/apache/tika/metadata/Message.java | 2 +-
.../java/org/apache/tika/metadata/Metadata.java | 328 ++++-
.../main/java/org/apache/tika/metadata/Office.java | 2 +-
.../apache/tika/metadata/OfficeOpenXMLCore.java | 7 -
.../tika/metadata/OfficeOpenXMLExtended.java | 2 +-
.../java/org/apache/tika/metadata/OneNote.java | 63 +
.../main/java/org/apache/tika/metadata/PDF.java | 7 +-
.../apache/tika/metadata/PassthroughPrefix.java | 79 --
.../java/org/apache/tika/metadata/Photoshop.java | 2 +-
.../java/org/apache/tika/metadata/Property.java | 227 +++-
.../java/org/apache/tika/metadata/RTFMetadata.java | 2 +-
.../java/org/apache/tika/metadata/Rendering.java | 18 +-
.../apache/tika/metadata/ReservedNamespaces.java | 10 +-
.../main/java/org/apache/tika/metadata/TIFF.java | 4 +-
.../apache/tika/metadata/TikaCoreProperties.java | 156 ++-
.../org/apache/tika/metadata/TikaPagedText.java | 6 +-
.../main/java/org/apache/tika/metadata/XMPDM.java | 84 +-
.../main/java/org/apache/tika/metadata/XMPMM.java | 2 +-
.../main/java/org/apache/tika/metadata/Zip.java | 2 +-
.../metadata/filter/LegacyKeyMigrationFilter.java | 175 ++-
.../filter/RemoveByMimeMetadataFilter.java | 3 +-
.../MetadataWriteLimiter.java | 2 +-
.../MetadataWriteLimiterFactory.java | 4 +-
.../StandardMetadataLimiter.java | 56 +-
.../StandardMetadataLimiterFactory.java | 2 +-
.../main/java/org/apache/tika/mime/MimeTypes.java | 3 +-
.../mime/ProbabilisticMimeDetectionSelector.java | 3 +-
.../org/apache/tika/parser/AutoDetectParser.java | 3 +-
.../org/apache/tika/parser/CompositeParser.java | 19 +-
.../java/org/apache/tika/parser/NetworkParser.java | 40 +-
.../java/org/apache/tika/parser/ParseContext.java | 2 +-
.../java/org/apache/tika/parser/ParseRecord.java | 30 +
.../apache/tika/parser/RecursiveParserWrapper.java | 3 +-
.../tika/parser/external/ExternalParser.java | 13 +-
.../tika/parser/external/ExternalParserConfig.java | 10 +-
.../parser/multiple/AbstractMultipleParser.java | 3 +-
.../sax/AbstractRecursiveParserWrapperHandler.java | 6 -
.../tika/sax/BasicContentHandlerFactory.java | 20 +-
.../org/apache/tika/sax/DIFContentHandler.java | 150 --
.../tika/sax/RecursiveParserWrapperHandler.java | 2 -
.../org/apache/tika/sax/XHTMLContentHandler.java | 3 +-
.../java/org/apache/tika/utils/ExceptionUtils.java | 22 -
.../org/apache/tika/utils/FileProcessResult.java | 39 +
.../java/org/apache/tika/utils/ParserUtils.java | 18 +-
.../java/org/apache/tika/utils/ProcessUtils.java | 167 ++-
.../tika/metadata/metadata-migration-3x-4x.json | 115 +-
.../org/apache/tika/config/ParseTimeoutTest.java | 267 ++++
.../tika/config/TikaProgressTrackerTest.java | 103 --
.../org/apache/tika/config/TimeoutLimitsTest.java | 47 +
.../tika/detect/FileCommandDetectorTest.java | 26 +
.../tika/detect/MetadataCharsetDetectorTest.java | 13 +-
.../org/apache/tika/detect/TypeDetectorTest.java | 3 +-
.../EmbeddedDocumentUtilExtensionTest.java | 9 +-
...arsingEmbeddedDocumentExtractorTimeoutTest.java | 277 ++++
.../java/org/apache/tika/io/FilenameUtilsTest.java | 3 +-
.../org/apache/tika/io/TikaInputStreamTest.java | 11 +-
.../org/apache/tika/metadata/KeyPrefixTest.java | 126 ++
.../metadata/MetadataInternalKeyGuardTest.java | 96 +-
.../tika/metadata/MetadataKeyPrefixRouteTest.java | 130 ++
.../apache/tika/metadata/MetadataPutAllTest.java | 141 ++
.../tika/metadata/MetadataTypedAccessorsTest.java | 117 ++
.../tika/metadata/PropertyReservedNameTest.java | 161 +++
.../org/apache/tika/metadata/PropertyTest.java | 113 ++
.../org/apache/tika/metadata/TestMetadata.java | 61 +-
.../filter/LegacyKeyMigrationFilterTest.java | 131 ++
.../org/apache/tika/mime/MimeDetectionTest.java | 3 +-
.../tika/mime/ProbabilisticMimeDetectionTest.java | 5 +-
.../ProbabilisticMimeDetectionTestWithTika.java | 3 +-
.../apache/tika/parser/CompositeParserTest.java | 61 +-
.../org/apache/tika/parser/NetworkParserTest.java | 135 ++
.../org/apache/tika/parser/mock/MockParser.java | 28 +-
.../tika/parser/multiple/MultipleParserTest.java | 5 +-
.../tika/sax/BasicContentHandlerFactoryTest.java | 33 +
.../org/apache/tika/utils/ProcessUtilsTest.java | 136 ++
.../apache/tika/detect/magika/MagikaDetector.java | 18 +-
.../tika/detect/magika/TestMagikaIntegration.java | 3 +-
.../tika/detect/siegfried/SiegfriedDetector.java | 18 +-
.../resources/tika-config-ignite-handlertype.json | 8 +-
.../test/resources/tika-config-ignite-local.json | 7 +-
.../src/test/resources/tika-config-ignite.json | 7 +-
.../tika-grpc/src/test/resources/tika-config.json | 7 +-
.../apache/tika/server/e2e/RunUatSmokeTest.java | 6 +-
.../tika/server/e2e/TikaServerHttp2Test.java | 6 +-
.../html/StandardHtmlEncodingDetectorTest.java | 5 +-
.../tika/detect/icu4j/Icu4jEncodingDetector.java | 5 +-
.../ml/chardetect/MojibusterEncodingDetector.java | 3 +-
.../universal/UniversalEncodingListener.java | 5 +-
.../org/apache/tika/eval/app/ProfilerBase.java | 16 +-
.../org/apache/tika/eval/app/io/ExtractReader.java | 15 +-
.../apache/tika/eval/app/io/ExtractReaderTest.java | 3 +-
.../src/test/resources/extracts/TIKA-4446-a.json | 2 +-
.../src/test/resources/extracts/TIKA-4446-b.json | 2 +-
.../resources/test-dirs/extractsA/file15_tags.json | 2 +-
.../test-dirs/extractsA/file16_badTags.json | 2 +-
.../test-dirs/extractsA/file17_tagsOutOfOrder.json | 2 +-
.../test/resources/test-dirs/raw_input/file15_tags | 2 +-
.../resources/test-dirs/raw_input/file16_badTags | 2 +-
.../test-dirs/raw_input/file17_tagsOutOfOrder | 2 +-
.../tika/eval/core/util/EvalExceptionUtils.java | 16 +-
.../org/apache/tika/example/ImportContextImpl.java | 3 +-
.../tika/example/MetadataAwareLuceneIndexer.java | 2 +-
.../java/org/apache/tika/example/MyFirstTika.java | 3 +-
.../tika/example/PipesForkParserExample.java | 11 +-
.../org/apache/tika/example/TIAParsingExample.java | 3 +-
.../apache/tika/pipes/grpc/TikaGrpcServerImpl.java | 1 +
.../apache/tika/pipes/grpc/TikaGrpcServerTest.java | 71 +
.../src/test/resources/tika-pipes-test-config.json | 4 +-
tika-metadata-schema/README.md | 29 +-
tika-metadata-schema/pom.xml | 37 +-
tika-metadata-schema/regen.sh | 34 +-
.../tika/metadata/schema/MetadataKeyValidator.java | 81 +-
.../metadata/schema/MigrationTableGenerator.java | 95 +-
.../tika/metadata/schema/SchemaGenerator.java | 140 +-
.../apache/tika/metadata/metadata-key-fields.json | 142 +-
.../org/apache/tika/metadata/metadata-keys.json | 1426 +++++++++++---------
.../tika/metadata/metadata-open-namespaces.json | 61 +-
.../schema/MetadataChangesAdocDriftTest.java | 122 ++
.../tika/metadata/schema/MetadataCoverageTest.java | 16 +-
.../metadata/schema/MetadataKeyValidatorTest.java | 10 +
.../schema/MetadataMigrationTableTest.java | 4 +-
.../metadata/schema/MetadataNoUnderscoreTest.java | 18 +-
.../tika/metadata/schema/MetadataSchemaTest.java | 4 +-
.../metadata/schema/MigrationCompletenessTest.java | 119 ++
.../org/apache/tika/metadata/migration-overlay.tsv | 223 ++-
.../forbidden-apis-metadata-string-key-bans.txt | 32 +
tika-parent/pom.xml | 19 +-
.../java/org/apache/tika/http/TikaHttpClient.java | 185 ++-
.../org/apache/tika/http/TikaHttpClientTest.java | 185 +++
.../org/apache/tika/http/TikaTestHttpServer.java | 38 +-
.../tika/parser/ocrencode/EncodeOCRParser.java | 5 +-
.../tika-parser-scientific-module/pom.xml | 15 +
.../tika/parser/climate/ClimateForecast.java | 64 +
.../apache/tika/parser/envi/EnviHeaderParser.java | 19 +-
.../org/apache/tika/parser/gdal/GDALParser.java | 86 +-
.../geoinfo/GeographicInformationParser.java | 101 +-
.../org/apache/tika/parser/grib/GribParser.java | 33 +-
.../java/org/apache/tika/parser/hdf/HDFParser.java | 17 +-
.../org/apache/tika/parser/isatab/ISATabUtils.java | 14 +-
.../apache/tika/parser/isatab/ISArchiveParser.java | 9 +-
.../apache/tika/parser/netcdf/NetCDFParser.java | 35 +-
.../apache/tika/parser/gdal/TestGDALParser.java | 70 +-
.../geoinfo/GeographicInformationParserTest.java | 12 +-
.../org/apache/tika/parser/hdf/HDFParserTest.java | 8 +-
.../tika/parser/isatab/ISArchiveParserTest.java | 15 +-
.../tika/parser/netcdf/NetCDFParserTest.java | 13 +-
.../tika/inference/AbstractEmbeddingFilter.java | 23 +-
.../tika/inference/ImageEmbeddingConfig.java | 10 +-
.../org/apache/tika/inference/InferenceConfig.java | 12 +-
.../tika/inference/OpenAIEmbeddingFilter.java | 7 +-
.../tika/inference/OpenAIImageEmbeddingParser.java | 26 +-
.../tika/inference/JinaEmbeddingFilterTest.java | 2 +-
.../tika/inference/OpenAIEmbeddingFilterTest.java | 2 +-
.../inference/OpenAIImageEmbeddingParserTest.java | 21 +-
.../tika-parsers-ml/tika-parser-nlp-module/pom.xml | 15 +
.../tika/parser/ctakes/CTAKESContentHandler.java | 10 +-
.../apache/tika/parser/geo/topic/GeoParser.java | 31 +-
.../tika/parser/journal/GrobidRESTParser.java | 21 +-
.../apache/tika/parser/journal/TEIDOMParser.java | 41 +-
.../apache/tika/parser/ner/NamedEntityParser.java | 18 +-
.../tika/parser/geo/topic/GeoParserTest.java | 24 +-
.../tika/parser/journal/JournalParserTest.java | 2 +-
.../org/apache/tika/parser/journal/TEITest.java | 13 +-
.../tika/parser/ner/NamedEntityParserTest.java | 12 +-
.../tika/parser/ner/nltk/NLTKNERecogniserTest.java | 2 +-
.../parser/ner/regex/RegexNERecogniserTest.java | 2 +-
.../tika/parser/ocr/tess4j/Tess4JConfig.java | 19 +-
.../tika/parser/ocr/tess4j/Tess4JParser.java | 155 ++-
.../tika/parser/ocr/tess4j/Tess4JConfigTest.java | 2 +-
.../parser/ocr/tess4j/Tess4JOCRTimeoutTest.java | 250 ++++
.../tika/parser/ocr/tess4j/Tess4JParserTest.java | 6 +-
.../resources/config-examples/tess4j-full.json | 2 +-
.../apache/tika/parser/vlm/AbstractVLMParser.java | 24 +-
.../org/apache/tika/parser/vlm/VLMOCRConfig.java | 12 +-
.../tika/parser/vlm/ClaudeVLMParserTest.java | 13 +-
.../tika/parser/vlm/GeminiVLMParserTest.java | 13 +-
.../tika/parser/vlm/OpenAIVLMParserTest.java | 19 +-
.../resources/config-examples/claude-vlm-full.json | 2 +-
.../resources/config-examples/gemini-vlm-full.json | 2 +-
.../config-examples/openai-vlm-basic.json | 2 +-
.../resources/config-examples/openai-vlm-full.json | 2 +-
.../tika/detect/TestContainerAwareDetector.java | 3 +-
.../java/org/apache/tika/mime/OneOffMimeTest.java | 5 +-
.../apache/tika/parser/AndroidBinaryXMLTest.java | 7 +-
.../tika/parser/AutoDetectParserConfigTest.java | 3 +-
.../apache/tika/parser/AutoDetectParserTest.java | 11 +-
.../tika/parser/AutoDetectReaderParserTest.java | 3 +-
.../tika/parser/RecursiveParserWrapperTest.java | 7 +-
.../apache/tika/parser/apple/PListParserTest.java | 3 +-
.../tika/parser/crypto/Pkcs7EndToEndTest.java | 7 +-
.../apache/tika/parser/crypto/TSDParserTest.java | 3 +-
.../tika/parser/digest/DigestConfigTest.java | 5 +-
.../parser/html/HtmlEncodingDetectionTest.java | 11 +-
.../apache/tika/parser/html/HtmlParserTest.java | 5 +-
.../apache/tika/parser/image/JpegParserTest.java | 5 +-
.../apache/tika/parser/mail/MboxParserTest.java | 3 +-
.../apache/tika/parser/mail/RFC822ParserTest.java | 10 +-
.../tika/parser/microsoft/EMFParserTest.java | 7 +-
.../tika/parser/microsoft/XML2003ParserTest.java | 3 +-
.../microsoft/ooxml/OleObjectRegressionTest.java | 5 +-
.../parser/microsoft/ooxml/TruncatedOOXMLTest.java | 5 +-
.../tika/parser/microsoft/rtf/RTFParserTest.java | 3 +-
.../tika/parser/ocr/TesseractOCRParserTest.java | 21 +-
.../org/apache/tika/parser/odf/ODFParserTest.java | 21 +-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 16 +-
.../org/apache/tika/parser/pkg/ArParserTest.java | 5 +-
.../apache/tika/parser/pkg/Bzip2ParserTest.java | 3 +-
.../apache/tika/parser/pkg/CompressParserTest.java | 3 +-
.../tika/parser/pkg/CompressorParserTest.java | 5 +-
.../org/apache/tika/parser/pkg/GzipParserTest.java | 5 +-
.../org/apache/tika/parser/pkg/RarParserTest.java | 3 +-
.../apache/tika/parser/pkg/Seven7ParserTest.java | 5 +-
.../org/apache/tika/parser/pkg/TarParserTest.java | 3 +-
.../org/apache/tika/parser/pkg/ZipParserTest.java | 3 +-
.../org/apache/tika/parser/pkg/ZlibParserTest.java | 3 +-
.../config-examples/migration-full-example.json | 2 +-
.../resources/config-examples/tesseract-basic.json | 2 +-
.../resources/config-examples/tesseract-full.json | 2 +-
.../tika-parser-apple-module/pom.xml | 15 +
.../org/apache/tika/parser/apple/PListParser.java | 5 +-
.../tika/parser/iwork/IWorkPackageParser.java | 3 +-
.../tika/parser/iwork/KeynoteContentHandler.java | 8 +-
.../tika/parser/iwork/NumbersContentHandler.java | 14 +-
.../tika/parser/iwork/PagesContentHandler.java | 36 +-
.../parser/iwork/iwana/IWork13PackageParser.java | 13 +-
.../parser/iwork/iwana/IWork18PackageParser.java | 3 +-
.../apache/tika/parser/apple/PListParserTest.java | 3 +-
.../apache/tika/parser/iwork/IWorkParserTest.java | 16 +-
.../tika/parser/iwork/iwana/IWork13ParserTest.java | 15 +-
.../tika-parser-audiovideo-module/pom.xml | 15 +
.../org/apache/tika/parser/audio/AudioParser.java | 59 +-
.../org/apache/tika/parser/audio/MidiParser.java | 25 +-
.../org/apache/tika/parser/mp3/ID3v2Frame.java | 16 +-
.../java/org/apache/tika/parser/mp3/Mp3Parser.java | 15 +-
.../java/org/apache/tika/parser/mp4/MP4Parser.java | 44 +-
.../apache/tika/parser/mp4/TikaMp4BoxHandler.java | 7 +-
.../org/apache/tika/parser/mp4/TikaMp4Reader.java | 149 ++
.../tika/parser/mp4/TikaMp4SoundHandler.java | 15 +-
.../tika/parser/mp4/boxes/TikaUserDataBox.java | 174 ++-
.../org/apache/tika/parser/ogg/FlacParser.java | 10 +-
.../org/apache/tika/parser/ogg/OggAudioParser.java | 27 +-
.../java/org/apache/tika/parser/ogg/OggParser.java | 12 +-
.../org/apache/tika/parser/ogg/OpusParser.java | 6 +-
.../org/apache/tika/parser/ogg/SpeexParser.java | 6 +-
.../org/apache/tika/parser/ogg/TheoraParser.java | 5 +-
.../org/apache/tika/parser/ogg/VorbisParser.java | 5 +-
.../org/apache/tika/parser/video/FLVParser.java | 71 +-
.../apache/tika/parser/audio/AudioParserTest.java | 57 +-
.../apache/tika/parser/audio/MidiParserTest.java | 9 +-
.../org/apache/tika/parser/mp3/ID3v2FrameTest.java | 13 +
.../org/apache/tika/parser/mp3/Mp3ParserTest.java | 99 +-
.../org/apache/tika/parser/mp4/MP4ParserTest.java | 150 +-
.../tika/parser/mp4/boxes/TikaUserDataBoxTest.java | 61 +
.../org/apache/tika/parser/ogg/FlacParserTest.java | 9 +-
.../apache/tika/parser/ogg/OggAudioParserTest.java | 3 +-
.../apache/tika/parser/ogg/VorbisParserTest.java | 9 +-
.../apache/tika/parser/video/FLVParserTest.java | 25 +-
.../tika-parser-cad-module/pom.xml | 15 +
.../java/org/apache/tika/parser/dwg/DWGParser.java | 19 +-
.../apache/tika/parser/dwg/DWGParserConfig.java | 10 +-
.../org/apache/tika/parser/dwg/DWGReadParser.java | 13 +-
.../java/org/apache/tika/parser/prt/PRTParser.java | 3 +-
.../org/apache/tika/parser/dwg/DWGParserTest.java | 14 +-
.../org/apache/tika/parser/prt/PRTParserTest.java | 5 +-
.../configs/tika-config-dwgRead-Timeout.json | 2 +-
.../apache/tika/parser/code/SourceCodeParser.java | 7 +-
.../tika/parser/executable/ExecutableParser.java | 37 +-
.../java/org/apache/tika/parser/mat/MatParser.java | 3 +-
.../org/apache/tika/parser/sas/SAS7BDATParser.java | 2 +-
.../tika/parser/code/SourceCodeParserTest.java | 3 +-
.../parser/executable/ExecutableParserTest.java | 9 +-
.../executable/UniversalExecutableParserTest.java | 5 +-
.../apache/tika/parser/sas/SAS7BDATParserTest.java | 4 +-
.../org/apache/tika/parser/crypto/Pkcs7Parser.java | 7 +-
.../org/apache/tika/parser/crypto/TSDParser.java | 3 +-
.../apache/tika/parser/crypto/Pkcs7ParserTest.java | 3 +-
.../tika-parser-font-module/pom.xml | 15 +
.../tika/parser/font/AdobeFontMetricParser.java | 6 +-
.../apache/tika/parser/font/TrueTypeParser.java | 18 +-
.../apache/tika/parser/font/FontParsersTest.java | 5 +-
.../org/apache/tika/parser/html/HtmlHandler.java | 28 +-
.../org/apache/tika/parser/html/JSoupParser.java | 7 +-
.../apache/tika/parser/html/HtmlParserTest.java | 47 +-
.../tika-parser-image-module/pom.xml | 15 +
.../tika/parser/image/AbstractImageParser.java | 9 +-
.../org/apache/tika/parser/image/ICNSParser.java | 12 +-
.../tika/parser/image/ImageMetadataExtractor.java | 82 +-
.../org/apache/tika/parser/image/ImageParser.java | 23 +-
.../org/apache/tika/parser/image/HeicXmpTest.java | 5 +-
.../apache/tika/parser/image/HeifParserTest.java | 7 +-
.../apache/tika/parser/image/ICNSParserTest.java | 5 +-
.../apache/tika/parser/image/ImageParserTest.java | 34 +-
.../apache/tika/parser/image/JpegParserTest.java | 133 +-
.../tika/parser/image/MotionPhotoXmpTest.java | 5 +-
.../apache/tika/parser/image/PSDParserTest.java | 20 +-
.../apache/tika/parser/image/WebPParserTest.java | 5 +-
.../org/apache/tika/parser/image/WebPXmpTest.java | 5 +-
.../apache/tika/parser/jdbc/JDBCTableReader.java | 5 +-
.../tika-parser-mail-module/pom.xml | 15 +
.../tika/parser/mail/MailContentHandler.java | 29 +-
.../org/apache/tika/parser/mbox/MboxParser.java | 34 +-
.../apache/tika/parser/mail/RFC822ParserTest.java | 67 +-
.../apache/tika/parser/mbox/MboxParserTest.java | 28 +-
.../tika-parser-microsoft-module/pom.xml | 15 +
.../microsoft/PSTEmailStreamTranslator.java | 3 +-
.../parser/microsoft/AbstractPOIFSExtractor.java | 15 +-
.../apache/tika/parser/microsoft/EMFParser.java | 3 +-
.../tika/parser/microsoft/JackcessExtractor.java | 9 +-
.../tika/parser/microsoft/JackcessParser.java | 14 +-
.../apache/tika/parser/microsoft/OfficeParser.java | 7 +-
.../tika/parser/microsoft/OutlookExtractor.java | 12 +-
.../tika/parser/microsoft/SummaryExtractor.java | 36 +-
.../apache/tika/parser/microsoft/TNEFParser.java | 3 +-
.../tika/parser/microsoft/chm/ChmParser.java | 3 +-
.../tika/parser/microsoft/libpst/LibPstParser.java | 7 +-
.../microsoft/libpst/LibPstParserConfig.java | 10 +-
.../microsoft/msg/ExtendedMetadataExtractor.java | 15 +-
.../parser/microsoft/onenote/OneNoteParser.java | 44 +-
.../onenote/fsshttpb/MSOneStorePackage.java | 91 +-
.../onenote/fsshttpb/MSOneStoreParser.java | 1 +
.../streamobj/RevisionStoreObjectGroup.java | 3 +
.../microsoft/ooxml/AbstractOOXMLExtractor.java | 5 +-
.../microsoft/ooxml/OOXMLExtractorFactory.java | 5 +-
.../microsoft/ooxml/SAXBasedMetadataExtractor.java | 50 +-
.../parser/microsoft/pst/OutlookPSTParser.java | 7 +-
.../parser/microsoft/pst/PSTMailItemParser.java | 7 +-
.../parser/microsoft/rtf/RTFEmbObjHandler.java | 8 +-
.../parser/microsoft/rtf/RTFObjDataParser.java | 3 +-
.../tika/parser/microsoft/rtf/RTFParser.java | 3 +-
.../microsoft/rtf/jflex/RTFEmbeddedHandler.java | 8 +-
.../rtf/jflex/RTFObjDataStreamParser.java | 3 +-
.../parser/microsoft/xml/SpreadsheetMLParser.java | 3 +-
.../tika/parser/microsoft/xml/WordMLParser.java | 3 +-
.../tika/parser/microsoft/EMFParserTest.java | 3 +-
.../tika/parser/microsoft/ExcelParserTest.java | 82 +-
.../tika/parser/microsoft/OldExcelParserTest.java | 3 +-
.../tika/parser/microsoft/OutlookParserTest.java | 15 +-
.../microsoft/POIContainerExtractionTest.java | 7 +-
.../parser/microsoft/PowerPointParserTest.java | 13 +-
.../tika/parser/microsoft/ProjectParserTest.java | 3 +-
.../tika/parser/microsoft/PublisherParserTest.java | 3 +-
.../parser/microsoft/SolidworksParserTest.java | 13 +-
.../tika/parser/microsoft/VisioParserTest.java | 3 +-
.../tika/parser/microsoft/WMFParserTest.java | 3 +-
.../tika/parser/microsoft/WordParserTest.java | 21 +-
.../microsoft/activemime/ActiveMimeParserTest.java | 3 +-
.../parser/microsoft/libpst/TestLibPstParser.java | 3 +-
.../onenote/fsshttpb/MSOneStorePackageTest.java | 1 -
.../ooxml/OOXMLContainerExtractionTest.java | 3 +-
.../parser/microsoft/ooxml/OOXMLDocxSAXTest.java | 21 +-
.../parser/microsoft/ooxml/OOXMLParserTest.java | 25 +-
.../parser/microsoft/ooxml/OOXMLPptxSAXTest.java | 25 +-
.../parser/microsoft/ooxml/TruncatedOOXMLTest.java | 5 +-
.../parser/microsoft/ooxml/VSDXParserTest.java | 3 +-
.../parser/microsoft/ooxml/xps/XPSParserTest.java | 9 +-
.../parser/microsoft/pst/OutlookPSTParserTest.java | 3 +-
.../tika/parser/microsoft/rtf/RTFParserTest.java | 5 +-
.../parser/microsoft/xml/XML2003ParserTest.java | 5 +-
.../tika-parser-miscoffice-module/pom.xml | 15 +
.../java/org/apache/tika/parser/dbf/DBFParser.java | 5 +-
.../apache/tika/parser/dif/DIFContentHandler.java | 9 +-
.../org/apache/tika/parser/epub/EpubParser.java | 5 +-
.../apache/tika/parser/hwp/HwpTextExtractorV5.java | 3 +-
.../org/apache/tika/parser/hwp/HwpV5Parser.java | 3 +-
.../indesign/ContentAndMetadataExtractor.java | 22 +-
.../apache/tika/parser/indesign/IDMLParser.java | 12 +-
.../apache/tika/parser/mif/MIFContentHandler.java | 7 +-
.../java/org/apache/tika/parser/mif/MIFParser.java | 5 +-
.../tika/parser/odf/FlatOpenDocumentParser.java | 3 +-
.../tika/parser/odf/OpenDocumentMetaParser.java | 17 +-
.../apache/tika/parser/odf/OpenDocumentParser.java | 5 +-
.../tika/parser/wordperfect/QuattroProParser.java | 5 +-
.../tika/parser/wordperfect/WordPerfectParser.java | 13 +-
.../org/apache/tika/parser/dbf/DBFParserTest.java | 11 +-
.../org/apache/tika/parser/dif/DIFParserTest.java | 6 +-
.../apache/tika/parser/epub/EpubParserTest.java | 5 +-
.../apache/tika/parser/hwp/HwpV5ParserTest.java | 5 +-
.../tika/parser/ibooks/iBooksParserTest.java | 4 +-
.../tika/parser/indesign/IDMLParserTest.java | 8 +-
.../org/apache/tika/parser/mif/MIFParserTest.java | 3 +-
.../org/apache/tika/parser/odf/ODFParserTest.java | 19 +-
.../tika/parser/wordperfect/QuattroProTest.java | 8 +-
.../tika/parser/wordperfect/WordPerfectTest.java | 14 +-
.../apache/tika/parser/iptc/IptcAnpaParser.java | 3 +-
.../apache/tika/parser/ocr/TesseractOCRConfig.java | 16 +-
.../apache/tika/parser/ocr/TesseractOCRParser.java | 31 +-
.../tika/parser/ocr/TesseractOCRConfigTest.java | 6 +-
.../tika/parser/ocr/TesseractOCRParserTest.java | 2 +-
.../resources/configs/TIKA-3582-tesseract.json | 2 +-
.../configs/tika-config-tesseract-full.json | 2 +-
.../configs/tika-config-tesseract-partial.json | 2 +-
.../tika-parser-pdf-module/pom.xml | 15 +
.../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 44 +-
.../java/org/apache/tika/parser/pdf/PDFParser.java | 28 +-
.../tika/parser/pdf/PDMetadataExtractor.java | 53 +-
.../org/apache/tika/parser/pdf/XFAExtractor.java | 21 +-
.../tika/parser/pdf/image/ImageGraphicsEngine.java | 16 +-
.../tika/renderer/pdf/pdfbox/PDFBoxRenderer.java | 12 +-
.../tika/renderer/pdf/poppler/PopplerRenderer.java | 18 +-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 127 +-
.../tika/parser/pkg/AbstractArchiveParser.java | 3 +-
.../apache/tika/parser/pkg/CompressorParser.java | 3 +-
.../org/apache/tika/parser/pkg/PackageParser.java | 9 +-
.../java/org/apache/tika/parser/pkg/RarParser.java | 3 +-
.../org/apache/tika/parser/pkg/SevenZParser.java | 3 +-
.../org/apache/tika/parser/pkg/UnrarParser.java | 9 +-
.../java/org/apache/tika/parser/pkg/ZipParser.java | 11 +-
.../apache/tika/parser/pkg/AbstractPkgTest.java | 5 +-
.../tika/parser/pkg/CompressorParserTest.java | 3 +-
.../apache/tika/parser/csv/TextAndCSVParser.java | 19 +-
.../tika/parser/markdown/MarkdownParser.java | 7 +-
.../apache/tika/parser/strings/StringsConfig.java | 24 +-
.../apache/tika/parser/strings/StringsParser.java | 22 +-
.../java/org/apache/tika/parser/txt/TXTParser.java | 7 +-
.../tika/parser/csv/TextAndCSVParserTest.java | 29 +-
.../tika/parser/markdown/MarkdownParserTest.java | 9 +-
.../tika/parser/strings/StringsConfigTest.java | 6 +-
.../org/apache/tika/parser/txt/TXTParserTest.java | 49 +-
.../configs/tika-config-strings-full.json | 2 +-
.../configs/tika-config-strings-partial.json | 2 +-
.../tika-parser-webarchive-module/pom.xml | 15 +
.../org/apache/tika/parser/wacz/WACZParser.java | 3 +-
.../org/apache/tika/parser/warc/WARCParser.java | 21 +-
.../apache/tika/parser/http/HttpParserTest.java | 3 +-
.../apache/tika/parser/wacz/WACZParserTest.java | 5 +-
.../apache/tika/parser/warc/WARCParserTest.java | 11 +-
.../tika-parser-xml-module/pom.xml | 15 +
.../apache/tika/parser/tmx/TMXContentHandler.java | 26 +-
.../java/org/apache/tika/parser/tmx/TMXParser.java | 3 +-
.../tika/parser/xliff/XLIFF12ContentHandler.java | 14 +-
.../apache/tika/parser/xliff/XLIFF12Parser.java | 3 +-
.../tika/parser/xml/AbstractMetadataHandler.java | 33 +-
.../xml/AttributeDependantMetadataHandler.java | 25 +-
.../apache/tika/parser/xml/FictionBookParser.java | 3 +-
.../apache/tika/parser/xml/MetadataHandler.java | 16 +-
.../java/org/apache/tika/parser/xml/XMLParser.java | 5 +-
.../apache/tika/parser/xml/DcXMLParserTest.java | 3 +-
.../parser/xml/TextAndAttributeXMLParserTest.java | 3 +-
.../org/apache/tika/parser/xmp/XmpExtractor.java | 6 +-
.../org/apache/tika/async/cli/PluginsWriter.java | 10 +-
.../apache/tika/async/cli/SimpleAsyncConfig.java | 16 +-
.../org/apache/tika/async/cli/TikaAsyncCLI.java | 9 +-
.../apache/tika/async/cli/AsyncCliParserTest.java | 12 +-
.../apache/tika/async/cli/AsyncProcessorTest.java | 118 ++
.../tika/async/cli/TikaConfigAsyncWriterTest.java | 35 +
.../configs/config-content-only-default.json | 2 -
.../configs/config-content-only-dynamic.json | 2 -
.../test/resources/configs/config-template.json | 2 -
.../org/apache/tika/client/HttpClientFactory.java | 42 +-
.../org/apache/tika/pipes/api/PipesResult.java | 13 +-
.../tika/pipes/core/PerClientServerManager.java | 105 +-
.../org/apache/tika/pipes/core/PipesClient.java | 89 +-
.../org/apache/tika/pipes/core/PipesConfig.java | 153 ++-
.../org/apache/tika/pipes/core/ServerManager.java | 4 +-
.../tika/pipes/core/SharedServerManager.java | 10 +-
.../apache/tika/pipes/core/async/AsyncEmitter.java | 7 +
.../tika/pipes/core/async/AsyncProcessor.java | 20 +-
.../core/extractor/FrictionlessUnpackHandler.java | 3 +-
.../core/extractor/StandardUnpackSelector.java | 3 +-
.../tika/pipes/core/extractor/UnpackExtractor.java | 3 +-
.../tika/pipes/core/protocol/PipesMessage.java | 29 +-
.../serialization/FetchEmitTupleDeserializer.java | 17 +
.../core/serialization/JsonFetchEmitTupleList.java | 24 +-
.../tika/pipes/core/server/ConnectionHandler.java | 87 +-
.../apache/tika/pipes/core/server/EmitHandler.java | 43 +-
.../tika/pipes/core/server/FetchHandler.java | 3 +-
.../tika/pipes/core/server/ParseHandler.java | 11 +-
.../apache/tika/pipes/core/server/PipesServer.java | 156 ++-
.../apache/tika/pipes/core/server/PipesWorker.java | 2 +-
.../tika/pipes/core/server/ServerProtocolIO.java | 29 +
.../pipes/core/server/SharedServerResources.java | 2 +-
.../tika/pipes/core/TikaPipesConfigTest.java | 8 +
.../core/extractor/StandardUnpackSelectorTest.java | 3 +-
.../tika/pipes/core/protocol/PipesMessageTest.java | 4 +-
.../serialization/JsonFetchEmitTupleListTest.java | 28 +
.../UserMetadataReservedKeyStanceTest.java | 60 +
.../WireRestrictedFetchEmitTupleTest.java | 2 +-
.../core/server/EmitHandlerUserMetadataTest.java | 76 ++
.../pipes/core/server/ServerProtocolIOTest.java | 72 +
.../tika/pipes/fork/PipesForkParserTest.java | 5 +-
.../pipes/core/DigestingOpenContainersTest.java | 3 +-
.../apache/tika/pipes/core/EmbeddedLimitsTest.java | 6 +-
.../pipes/core/PartialTimeoutPassbackTest.java | 80 ++
.../apache/tika/pipes/core/PassbackFilterTest.java | 9 +-
.../apache/tika/pipes/core/PipesClientTest.java | 197 ++-
.../pipes/core/async/AsyncChaosMonkeyTest.java | 3 +-
.../configs/tika-config-timeout-lt-heartbeat.json | 4 +-
.../pipes/pipesiterator/PipesIteratorBase.java | 14 +-
.../tika/pipes/emitter/azblob/AZBlobEmitter.java | 5 +-
.../tika-pipes-plugins/tika-pipes-csv/pom.xml | 1 +
.../tika/pipes/iterator/csv/CSVPipesIterator.java | 14 +-
.../pipes/iterator/csv/TestCSVPipesIterator.java | 20 +
.../src/test/resources/test-reserved-header.csv | 2 +
.../tika/pipes/reporter/es/ESReporterConfig.java | 12 +-
.../apache/tika/pipes/es/ConfigExamplesTest.java | 18 +
.../tika/pipes/fetcher/http/HttpFetcher.java | 22 +-
.../fetcher/http/config/HttpFetcherConfig.java | 14 +
.../tika/pipes/fetcher/http/HttpFetcherTest.java | 24 +
.../pipes/iterator/jdbc/JDBCPipesIterator.java | 13 +-
.../pipes/iterator/jdbc/TestJDBCPipesIterator.java | 25 +-
.../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 3 +-
.../tika/config/loader/ComponentInstantiator.java | 5 +
.../tika/serialization/ComponentNameResolver.java | 2 +-
.../tika/serialization/ParseContextUtils.java | 20 +-
.../java/org/apache/tika/config/AllLimitsTest.java | 2 +-
.../org/apache/tika/config/TimeoutLimitsTest.java | 30 +-
.../apache/tika/config/loader/TikaLoaderTest.java | 17 +-
.../tika/metadata/filter/TestMetadataFilter.java | 69 +-
.../StandardMetadataLimiterTest.java | 96 +-
.../WireRestrictedParseContextTest.java | 14 +
.../TIKA-4816-legacy-key-migration-default.json | 5 +
.../configs/TIKA-4816-legacy-key-migration.json | 5 +
tika-server/README.md | 20 +-
.../sample-configs/vision/vlm-openai.json | 2 +-
tika-server/pom.xml | 1 -
tika-server/tika-server-client/pom.xml | 70 -
.../tika/server/client/TikaAsyncHttpClient.java | 37 -
.../org/apache/tika/server/client/TikaClient.java | 69 -
.../apache/tika/server/client/TikaClientCLI.java | 153 ---
.../server/client/TikaClientConfigException.java | 30 -
.../tika/server/client/TikaEmitterResult.java | 54 -
.../tika/server/client/TikaPipesHttpClient.java | 168 ---
.../tika/server/client/TikaServerClientConfig.java | 106 --
.../org/apache/tika/server/client/TestBasic.java | 55 -
.../src/test/resources/log4j2.xml | 32 -
.../resources/tika-config-simple-fs-emitter.json | 11 -
tika-server/tika-server-core/pom.xml | 43 -
.../server/core/BadRequestExceptionMapper.java | 48 +
.../tika/server/core/MaxRequestSizeFilter.java | 138 ++
.../org/apache/tika/server/core/ServerStatus.java | 2 +-
.../org/apache/tika/server/core/TikaServerCli.java | 7 +-
.../apache/tika/server/core/TikaServerConfig.java | 55 +-
.../core/TikaServerParseExceptionMapper.java | 6 +-
.../apache/tika/server/core/TikaServerProcess.java | 167 ++-
.../tika/server/core/resource/AsyncResource.java | 85 +-
.../server/core/resource/DetectorResource.java | 30 +-
.../server/core/resource/LanguageResource.java | 56 +-
.../server/core/resource/MetadataResource.java | 38 +-
.../server/core/resource/PipesParsingHelper.java | 313 +++--
.../tika/server/core/resource/PipesResource.java | 106 +-
.../core/resource/RecursiveMetadataResource.java | 8 +-
.../tika/server/core/resource/TikaResource.java | 401 ++----
.../server/core/resource/TikaResourceAware.java | 18 +-
.../tika/server/core/resource/TikaWelcome.java | 20 +-
.../server/core/resource/TranslateResource.java | 137 --
.../server/core/resource/UnpackerResource.java | 2 -
.../apache/tika/server/core/AsyncResourceTest.java | 242 ++++
.../org/apache/tika/server/core/CXFTestBase.java | 10 +-
.../core/ConfigEndpointSecurityEnabledTest.java | 20 +-
.../server/core/ConfigEndpointSecurityTest.java | 2 +-
.../tika/server/core/LanguageResourceTest.java | 89 +-
.../tika/server/core/MaxRequestSizeFilterTest.java | 127 ++
.../server/core/RecursiveMetadataResourceTest.java | 3 +-
.../apache/tika/server/core/StackTraceTest.java | 87 +-
.../org/apache/tika/server/core/TikaPipesTest.java | 112 +-
.../apache/tika/server/core/TikaResourceTest.java | 155 ++-
.../tika/server/core/TikaServerConfigTest.java | 9 +
.../server/core/TikaServerIntegrationTest.java | 19 +
.../core/TikaServerPipesIntegrationTest.java | 48 +-
.../tika/server/core/TikaServerProcessTest.java | 36 +
.../apache/tika/server/core/TikaWelcomeTest.java | 2 +-
.../tika/server/core/TranslateResourceTest.java | 88 --
.../core/resource/PipesResponseBuilderTest.java | 90 ++
.../core/resource/ReservedComponentIdTest.java | 79 ++
.../test-documents/mock/hello_world_heading.xml} | 23 +-
.../standard/resource/XMPMetadataResource.java | 21 +-
.../tika/server/standard/DetectorResourceTest.java | 10 +-
.../server/standard/JsonMaxFieldLengthTest.java | 2 +-
.../tika/server/standard/MetadataResourceTest.java | 60 +-
.../standard/RecursiveMetadataFilterTest.java | 5 +-
.../standard/RecursiveMetadataResourceTest.java | 25 +-
.../apache/tika/server/standard/TikaPipesTest.java | 4 +-
.../tika/server/standard/TikaResourceTest.java | 48 +-
.../UnpackerResourceConfigDisabledTest.java | 2 +-
.../tika/server/standard/UnpackerResourceTest.java | 2 +-
.../standard/UnpackerResourceWithConfigTest.java | 2 +-
.../resource/ResourceServiceLoadingTest.java | 103 ++
.../main/java/org/apache/tika/xmp/XMPMetadata.java | 6 +-
.../org/apache/tika/xmp/convert/TikaToXMP.java | 3 +-
.../java/org/apache/tika/xmp/TikaToXMPTest.java | 5 +-
655 files changed, 16180 insertions(+), 6567 deletions(-)
diff --cc CHANGES.txt
index e46059d1a1,d2f1ae5bcf..6563f45e4e
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -8,6 -111,6 +111,9 @@@ Release 4.0.0 - ??
OTHER CHANGES
++ * OneNote extraction now follows document order, omits superseded page
++ revisions, and extracts embedded object BLOBs (TIKA-4814).
++
* MagicDetector now compiles its regular expression once, in the
constructor, instead of recompiling it on every match (TIKA-4796).
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
index 8312bdb614,6c1355cb15..8b5a7eabbd
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
@@@ -36,19 -31,13 +35,19 @@@ import java.util.regex.Matcher
import java.util.regex.Pattern;
import org.xml.sax.SAXException;
+import org.xml.sax.helpers.AttributesImpl;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
+import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
- import org.apache.tika.metadata.Property;
+ import org.apache.tika.metadata.OneNote;
import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.microsoft.onenote.OneNotePropertyEnum;
import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.ArrayNumber;
import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.EightBytesOfData;
import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.FourBytesOfData;
import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.IProperty;
@@@ -91,11 -76,6 +90,7 @@@ public class MSOneStorePackage
private static final Pattern HYPERLINK_PATTERN =
Pattern.compile("\uFDDFHYPERLINK\\s+\"([^\"]+)\"([^\"]+)$");
private static final String P = "p";
- /**
- * A visible separator emitted between pages so they can be told apart in plain text
- * output.
- */
- private static final String PAGE_SEPARATOR = "----------------------------------------";
++ private static final int MAX_TRAVERSAL_DEPTH = 100;
static {
LocalDateTime time32Epoch1980 = LocalDateTime.of(1980, Month.JANUARY, 1, 0, 0);
@@@ -197,48 -162,93 +193,41 @@@
public void walkTree(OneNoteTreeWalkerOptions options, Metadata metadata,
XHTMLContentHandler xhtml)
throws SAXException, TikaException, IOException {
- for (RevisionStoreObjectGroup revisionStoreObjectGroup : OtherFileNodeList) {
- for (RevisionStoreObject revisionStoreObject : revisionStoreObjectGroup.objects) {
- PropertySet propertySet =
- revisionStoreObject.propertySet.objectSpaceObjectPropSet.body;
- for (int i = 0; i < propertySet.rgData.size(); ++i) {
- IProperty property = propertySet.rgData.get(i);
- PropertyID propertyID = propertySet.rgPrids[i];
- PropertyType propertyType = PropertyType.fromIntVal(propertyID.type);
- OneNotePropertyEnum oneNotePropertyEnum =
- OneNotePropertyEnum.of(Unsigned.uint(propertyID.value).longValue());
- if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTimeStamp) {
- long fullval = getScalar(property);
- Instant instant = Instant.ofEpochSecond(
- fullval / 10000000 + DATETIME_EPOCH_DIFF_1601);
- if (instant.isAfter(lastModifiedTimestamp)) {
- lastModifiedTimestamp = instant;
- }
- metadata.set(OneNote.LAST_MODIFIED_TIMESTAMP,
- String.valueOf(lastModifiedTimestamp.toEpochMilli()));
- } else if (oneNotePropertyEnum == OneNotePropertyEnum.CreationTimeStamp) {
- // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not
- // 1970
- long scalar = getScalar(property);
- long creationTs = scalar + TIME32_EPOCH_DIFF_1980;
- if (creationTs < creationTimestamp) {
- creationTimestamp = creationTs;
- }
- metadata.set(OneNote.CREATION_TIMESTAMP, String.valueOf(creationTimestamp));
- } else if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTime) {
- // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not
- // 1970
- long scalar = getScalar(property);
- long lastMod = scalar + TIME32_EPOCH_DIFF_1980;
- if (lastMod > lastModified) {
- lastModified = lastMod;
- }
- metadata.set(TikaCoreProperties.MODIFIED, String.valueOf(lastModified));
- } else if (oneNotePropertyEnum == OneNotePropertyEnum.Author) {
- String author =
- new String(((PrtFourBytesOfLengthFollowedByData) property).data,
- StandardCharsets.UTF_8);
- if (mostRecentAuthorProp) {
- mostRecentAuthors.add(author);
- } else if (originalAuthorProp) {
- originalAuthors.add(author);
- } else {
- authors.add(author);
- }
- } else if (oneNotePropertyEnum == OneNotePropertyEnum.AuthorMostRecent) {
- mostRecentAuthorProp = true;
- } else if (oneNotePropertyEnum == OneNotePropertyEnum.AuthorOriginal) {
- originalAuthorProp = true;
- } else if (propertyType == PropertyType.FourBytesOfLengthFollowedByData) {
- boolean isBinary = propertyIsBinary(oneNotePropertyEnum);
- PrtFourBytesOfLengthFollowedByData dataProperty =
- (PrtFourBytesOfLengthFollowedByData) property;
- if ((dataProperty.data.length & 1) == 0 &&
- oneNotePropertyEnum != OneNotePropertyEnum.TextExtendedAscii &&
- !isBinary) {
- if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) {
- xhtml.startElement(P);
- xhtml.characters(
- new String(dataProperty.data, StandardCharsets.UTF_16LE));
- xhtml.endElement(P);
- }
- } else if (oneNotePropertyEnum == OneNotePropertyEnum.TextExtendedAscii) {
- xhtml.startElement(P);
- xhtml.characters(
- new String(dataProperty.data, StandardCharsets.US_ASCII));
- xhtml.endElement(P);
- } else if (!isBinary) {
- if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) {
- xhtml.startElement(P);
- xhtml.characters(
- new String(dataProperty.data, StandardCharsets.UTF_16LE));
- xhtml.endElement(P);
- }
- } else {
- if (oneNotePropertyEnum == OneNotePropertyEnum.RichEditTextUnicode) {
- handleRichEditTextUnicode(dataProperty.data, xhtml);
- } else {
- //TODO -- these seem to be somewhat broken font files and other
- //odds and ends...what are they and how should we process them?
- //handleEmbedded(content.size());
- }
- }
- }
+ walkTree(options, metadata, xhtml, new ParseContext());
+ }
+
+ public void walkTree(OneNoteTreeWalkerOptions options, Metadata metadata,
+ XHTMLContentHandler xhtml, ParseContext parseContext)
+ throws SAXException, TikaException, IOException {
+ this.parseContext = parseContext;
+ this.parentMetadata = metadata;
+ this.embeddedDocumentExtractor =
+ EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(parseContext);
+ if (!cells.isEmpty()) {
+ // Walk each page cell (object space) as a tree, starting from the root objects of
+ // its current revision and following the object references in property order. This
+ // emits the text in document order. The pages are walked in the order in which the
+ // section object space references them; cells that hold older versions of a page
+ // (the same object space in a different revision context) are skipped.
+ List<RevisionStoreCell> pageCells = new ArrayList<>();
+ List<RevisionStoreCell> otherCells = new ArrayList<>();
+ splitCells(pageCells, otherCells);
- boolean firstPage = true;
+ for (RevisionStoreCell cell : pageCells) {
- if (!firstPage) {
- xhtml.startElement(P);
- xhtml.characters(PAGE_SEPARATOR);
- xhtml.endElement(P);
- }
- firstPage = false;
+ xhtml.startElement("div", "class", "page");
+ walkCell(cell, options, metadata, xhtml);
+ xhtml.endElement("div");
+ }
+ for (RevisionStoreCell cell : otherCells) {
+ walkCell(cell, options, metadata, xhtml);
+ }
+ } else {
+ // no cell information available - walk the object groups in revision order
+ Map<ExGuid, RevisionStoreObject> objectsById = indexObjectsById(OtherFileNodeList);
+ Set<ExGuid> visited = new HashSet<>();
+ for (RevisionStoreObjectGroup objectGroup : OtherFileNodeList) {
+ for (RevisionStoreObject object : objectGroup.objects) {
+ walkObject(object, objectsById, visited, AuthorRole.NONE, options, metadata,
- xhtml);
++ xhtml, 0);
}
}
}
@@@ -255,415 -265,6 +244,435 @@@
}
}
+ /**
+ * Splits the cells into page cells, ordered as the section object space references them,
+ * and the remaining cells. A cell that holds an older version of a page - the same object
+ * space referenced by a page cell, but in a different revision context - is dropped, so
+ * content is not emitted once per version snapshot.
+ */
+ private void splitCells(List<RevisionStoreCell> pageCells,
- List<RevisionStoreCell> otherCells) {
++ List<RevisionStoreCell> otherCells) throws TikaException {
+ List<CellID> orderedCellIds = collectSectionReferencedCells();
+ if (orderedCellIds.isEmpty()) {
+ // no page ordering information available - process the cells in storage order
+ pageCells.addAll(cells);
+ return;
+ }
+ Map<CellID, RevisionStoreCell> remainingCells = new LinkedHashMap<>();
+ for (RevisionStoreCell cell : cells) {
+ remainingCells.put(cell.cellID, cell);
+ }
+ Set<ExGuid> coveredObjectSpaces = new HashSet<>();
+ for (CellID cellId : orderedCellIds) {
+ RevisionStoreCell cell = remainingCells.remove(cellId);
+ if (cell != null) {
+ pageCells.add(cell);
+ coveredObjectSpaces.add(cellId.extendGUID2);
+ }
+ }
+ for (RevisionStoreCell cell : remainingCells.values()) {
+ if (cell.cellID == null ||
+ !coveredObjectSpaces.contains(cell.cellID.extendGUID2)) {
+ // not an older version of one of the pages - keep it so no content is lost
+ otherCells.add(cell);
+ }
+ }
+ }
+
+ /**
+ * Walks the section object space (the data root cell) and collects the object space (cell)
+ * references in document order - this is the order of the pages in the section.
+ */
- private List<CellID> collectSectionReferencedCells() {
++ private List<CellID> collectSectionReferencedCells() throws TikaException {
+ List<CellID> orderedCellIds = new ArrayList<>();
+ if (dataRootCell == null) {
+ return orderedCellIds;
+ }
+ Map<ExGuid, RevisionStoreObject> objectsById = indexObjectsById(dataRootCell.objectGroups);
+ Set<ExGuid> visited = new HashSet<>();
+ for (RevisionManifestRootDeclare rootDeclare : dataRootCell.rootDeclares) {
+ collectReferencedCells(objectsById.get(rootDeclare.objectExGuid), objectsById, visited,
- orderedCellIds);
++ orderedCellIds, 0);
+ }
+ for (RevisionStoreObjectGroup objectGroup : dataRootCell.objectGroups) {
+ for (RevisionStoreObject object : objectGroup.objects) {
- collectReferencedCells(object, objectsById, visited, orderedCellIds);
++ collectReferencedCells(object, objectsById, visited, orderedCellIds, 0);
+ }
+ }
+ return orderedCellIds;
+ }
+
+ private void collectReferencedCells(RevisionStoreObject object,
+ Map<ExGuid, RevisionStoreObject> objectsById,
- Set<ExGuid> visited, List<CellID> out) {
++ Set<ExGuid> visited, List<CellID> out, int depth)
++ throws TikaException {
++ if (depth > MAX_TRAVERSAL_DEPTH) {
++ return;
++ }
+ if (object == null || object.propertySet == null ||
+ object.propertySet.objectSpaceObjectPropSet == null) {
+ return;
+ }
+ if (object.objectID != null && !visited.add(object.objectID)) {
+ return;
+ }
+ List<PropertyAction> actions = collectObjectActions(object);
+ for (PropertyAction action : actions) {
+ if (action.spaceReference != null) {
+ out.add(action.spaceReference);
+ } else if (action.isChildReference && action.childReference != null) {
+ collectReferencedCells(objectsById.get(action.childReference), objectsById,
- visited, out);
++ visited, out, depth + 1);
+ }
+ }
+ }
+
+ private void walkCell(RevisionStoreCell cell, OneNoteTreeWalkerOptions options,
+ Metadata metadata, XHTMLContentHandler xhtml)
+ throws SAXException, TikaException, IOException {
+ Map<ExGuid, RevisionStoreObject> objectsById = indexObjectsById(cell.objectGroups);
+ Set<ExGuid> visited = new HashSet<>();
+ // Only objects reachable from the root objects of the current revision are part of
+ // the current content. The object groups may also contain older, superseded versions
+ // of objects (under a different object ID); those are intentionally not walked.
+ for (RevisionManifestRootDeclare rootDeclare : cell.rootDeclares) {
+ walkObject(objectsById.get(rootDeclare.objectExGuid), objectsById, visited,
- AuthorRole.NONE, options, metadata, xhtml);
++ AuthorRole.NONE, options, metadata, xhtml, 0);
+ }
+ if (visited.isEmpty()) {
+ // no root objects could be resolved - walk everything so no content is lost
+ for (RevisionStoreObjectGroup objectGroup : cell.objectGroups) {
+ for (RevisionStoreObject object : objectGroup.objects) {
+ walkObject(object, objectsById, visited, AuthorRole.NONE, options, metadata,
- xhtml);
++ xhtml, 0);
+ }
+ }
+ }
+ }
+
+ /**
+ * Builds a map of object ID to object. The object groups are ordered from the oldest
+ * revision to the newest, so a newer version of an object wins over an older one.
+ */
+ private Map<ExGuid, RevisionStoreObject> indexObjectsById(
+ List<RevisionStoreObjectGroup> objectGroups) {
+ Map<ExGuid, RevisionStoreObject> objectsById = new HashMap<>();
+ for (RevisionStoreObjectGroup objectGroup : objectGroups) {
+ for (RevisionStoreObject object : objectGroup.objects) {
+ if (object.objectID != null) {
+ objectsById.put(object.objectID, object);
+ }
+ }
+ }
+ return objectsById;
+ }
+
+ private void walkObject(RevisionStoreObject object,
+ Map<ExGuid, RevisionStoreObject> objectsById, Set<ExGuid> visited,
+ AuthorRole authorRole, OneNoteTreeWalkerOptions options,
- Metadata metadata, XHTMLContentHandler xhtml)
++ Metadata metadata, XHTMLContentHandler xhtml, int depth)
+ throws SAXException, TikaException, IOException {
++ if (depth > MAX_TRAVERSAL_DEPTH) {
++ throw new TikaException("OneNote object graph exceeds maximum depth of " +
++ MAX_TRAVERSAL_DEPTH);
++ }
+ if (object == null) {
+ return;
+ }
+ if (object.objectID != null && !visited.add(object.objectID)) {
+ return;
+ }
+ if (object.fileDataObject != null) {
+ // the object carries opaque binary data, e.g. an embedded image or file
+ handleEmbedded(object.fileDataObject.getData(), xhtml);
+ }
+ if (object.propertySet == null ||
+ object.propertySet.objectSpaceObjectPropSet == null) {
+ return;
+ }
+ List<PropertyAction> actions = collectObjectActions(object);
+ // An image node can reference the same picture twice: PictureContainer holds the
+ // canonical image data and WebPictureContainer14 holds a rendition derived from it
+ // (e.g. re-rendered when the picture was resized). Only emit the derived rendition
+ // when the canonical container is missing, so the picture is not extracted twice.
+ boolean hasPrimaryPicture = false;
+ for (PropertyAction action : actions) {
+ if (action.isChildReference && action.childReference != null &&
+ action.oneNotePropertyEnum == OneNotePropertyEnum.PictureContainer) {
+ hasPrimaryPicture = true;
+ break;
+ }
+ }
+ // The title structure of a page (StructureElementChildNodes) appears above the page
+ // body on screen, but is declared after the body child nodes. Emit it first so the
+ // text comes out in visual order.
+ for (PropertyAction action : actions) {
+ if (action.oneNotePropertyEnum == OneNotePropertyEnum.StructureElementChildNodes) {
- processAction(action, objectsById, visited, authorRole, options, metadata, xhtml);
++ processAction(action, objectsById, visited, authorRole, options, metadata, xhtml,
++ depth);
+ }
+ }
+ for (PropertyAction action : actions) {
+ if (hasPrimaryPicture &&
+ action.oneNotePropertyEnum == OneNotePropertyEnum.WebPictureContainer14) {
+ continue;
+ }
+ if (action.oneNotePropertyEnum != OneNotePropertyEnum.StructureElementChildNodes) {
- processAction(action, objectsById, visited, authorRole, options, metadata, xhtml);
++ processAction(action, objectsById, visited, authorRole, options, metadata, xhtml,
++ depth);
+ }
+ }
+ }
+
+ /**
+ * A property of an object, together with the object reference assigned to it if it is
+ * an object reference property.
+ */
+ private static final class PropertyAction {
+ private final IProperty property;
+ private final PropertyType propertyType;
+ private final OneNotePropertyEnum oneNotePropertyEnum;
+ private final boolean isChildReference;
+ private final ExGuid childReference;
+ private final CellID spaceReference;
+
+ PropertyAction(IProperty property, PropertyType propertyType,
+ OneNotePropertyEnum oneNotePropertyEnum, boolean isChildReference,
+ ExGuid childReference, CellID spaceReference) {
+ this.property = property;
+ this.propertyType = propertyType;
+ this.oneNotePropertyEnum = oneNotePropertyEnum;
+ this.isChildReference = isChildReference;
+ this.childReference = childReference;
+ this.spaceReference = spaceReference;
+ }
+ }
+
+ /**
+ * Flattens the properties of an object, in order, into a list of actions.
+ */
- private List<PropertyAction> collectObjectActions(RevisionStoreObject object) {
++ private List<PropertyAction> collectObjectActions(RevisionStoreObject object)
++ throws TikaException {
+ List<ExGuid> referencedObjects =
+ object.referencedObjectID == null || object.referencedObjectID.content == null ?
+ Collections.emptyList() : object.referencedObjectID.content;
+ List<CellID> referencedSpaces = object.referencedObjectSpacesID == null ||
+ object.referencedObjectSpacesID.content == null ? Collections.emptyList() :
+ object.referencedObjectSpacesID.content;
+ List<PropertyAction> actions = new ArrayList<>();
+ collectActions(object.propertySet.objectSpaceObjectPropSet.body, referencedObjects,
- new int[]{0}, referencedSpaces, new int[]{0}, actions);
++ new int[]{0}, referencedSpaces, new int[]{0}, actions, 0);
+ return actions;
+ }
+
+ /**
+ * Flattens the properties of a property set, in order, into a list of actions.
+ * Properties of type ObjectID or ArrayOfObjectIDs consume, in property order, the object
+ * references of the containing object, and properties of type ObjectSpaceID or
+ * ArrayOfObjectSpaceIDs consume the object space (cell) references (see MS-ONESTORE
+ * section 2.7.8), so the references must be assigned here, in property order, no matter
+ * in which order the actions are processed later.
+ */
+ private void collectActions(PropertySet propertySet, List<ExGuid> referencedObjects,
+ int[] referenceCursor, List<CellID> referencedSpaces,
- int[] spaceCursor, List<PropertyAction> actions) {
++ int[] spaceCursor, List<PropertyAction> actions, int depth)
++ throws TikaException {
++ if (depth > MAX_TRAVERSAL_DEPTH) {
++ throw new TikaException("OneNote property graph exceeds maximum depth of " +
++ MAX_TRAVERSAL_DEPTH);
++ }
+ if (propertySet == null || propertySet.rgPrids == null || propertySet.rgData == null) {
+ return;
+ }
+ for (int i = 0; i < propertySet.rgPrids.length && i < propertySet.rgData.size(); ++i) {
+ IProperty property = propertySet.rgData.get(i);
+ PropertyID propertyID = propertySet.rgPrids[i];
+ PropertyType propertyType = PropertyType.fromIntVal(propertyID.type);
+ OneNotePropertyEnum oneNotePropertyEnum =
+ OneNotePropertyEnum.of(Unsigned.uint(propertyID.value).longValue());
+ if (propertyType == PropertyType.ObjectID) {
+ actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, true,
+ nextReference(referencedObjects, referenceCursor), null));
+ } else if (propertyType == PropertyType.ArrayOfObjectIDs) {
- int count = property instanceof ArrayNumber ? ((ArrayNumber) property).number : 0;
++ int count = property instanceof ArrayNumber ?
++ Math.min(Math.max(0, ((ArrayNumber) property).number),
++ referencedObjects.size() - referenceCursor[0]) : 0;
+ for (int j = 0; j < count; ++j) {
+ actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum,
+ true, nextReference(referencedObjects, referenceCursor), null));
+ }
+ } else if (propertyType == PropertyType.ObjectSpaceID) {
+ actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, false,
+ null, nextSpaceReference(referencedSpaces, spaceCursor)));
+ } else if (propertyType == PropertyType.ArrayOfObjectSpaceIDs) {
- int count = property instanceof ArrayNumber ? ((ArrayNumber) property).number : 0;
++ int count = property instanceof ArrayNumber ?
++ Math.min(Math.max(0, ((ArrayNumber) property).number),
++ referencedSpaces.size() - spaceCursor[0]) : 0;
+ for (int j = 0; j < count; ++j) {
+ actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum,
+ false, null, nextSpaceReference(referencedSpaces, spaceCursor)));
+ }
+ } else if (propertyType == PropertyType.PropertySet) {
+ if (property instanceof PropertySet) {
+ collectActions((PropertySet) property, referencedObjects, referenceCursor,
- referencedSpaces, spaceCursor, actions);
++ referencedSpaces, spaceCursor, actions, depth + 1);
+ }
+ } else if (propertyType == PropertyType.ArrayOfPropertyValues) {
+ if (property instanceof PrtArrayOfPropertyValues &&
+ ((PrtArrayOfPropertyValues) property).data != null) {
+ for (PropertySet nested : ((PrtArrayOfPropertyValues) property).data) {
+ collectActions(nested, referencedObjects, referenceCursor,
- referencedSpaces, spaceCursor, actions);
++ referencedSpaces, spaceCursor, actions, depth + 1);
+ }
+ }
+ } else {
+ actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, false,
+ null, null));
+ }
+ }
+ }
+
+ private void processAction(PropertyAction action,
+ Map<ExGuid, RevisionStoreObject> objectsById, Set<ExGuid> visited,
+ AuthorRole authorRole, OneNoteTreeWalkerOptions options,
- Metadata metadata, XHTMLContentHandler xhtml)
++ Metadata metadata, XHTMLContentHandler xhtml, int depth)
+ throws SAXException, TikaException, IOException {
+ if (action.spaceReference != null) {
+ // a reference to another object space (cell) - cells are walked separately
+ return;
+ }
+ if (action.isChildReference) {
+ AuthorRole childRole = AuthorRole.NONE;
+ if (action.oneNotePropertyEnum == OneNotePropertyEnum.AuthorMostRecent) {
+ childRole = AuthorRole.MOST_RECENT;
+ } else if (action.oneNotePropertyEnum == OneNotePropertyEnum.AuthorOriginal) {
+ childRole = AuthorRole.ORIGINAL;
+ }
+ walkObject(action.childReference == null ? null :
+ objectsById.get(action.childReference), objectsById, visited,
- childRole, options, metadata, xhtml);
++ childRole, options, metadata, xhtml, depth + 1);
+ } else {
+ processPrimitiveProperty(action.property, action.propertyType,
+ action.oneNotePropertyEnum, authorRole, options, metadata, xhtml);
+ }
+ }
+
+ private ExGuid nextReference(List<ExGuid> referencedObjects, int[] referenceCursor) {
+ if (referenceCursor[0] < referencedObjects.size()) {
+ return referencedObjects.get(referenceCursor[0]++);
+ }
+ return null;
+ }
+
+ private CellID nextSpaceReference(List<CellID> referencedSpaces, int[] spaceCursor) {
+ if (spaceCursor[0] < referencedSpaces.size()) {
+ return referencedSpaces.get(spaceCursor[0]++);
+ }
+ return null;
+ }
+
+ private void processPrimitiveProperty(IProperty property, PropertyType propertyType,
+ OneNotePropertyEnum oneNotePropertyEnum,
+ AuthorRole authorRole,
+ OneNoteTreeWalkerOptions options, Metadata metadata,
+ XHTMLContentHandler xhtml)
+ throws SAXException, TikaException, IOException {
+ if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTimeStamp) {
+ long fullval = getScalar(property);
+ Instant instant = Instant.ofEpochSecond(
+ fullval / 10000000 + DATETIME_EPOCH_DIFF_1601);
+ if (instant.isAfter(lastModifiedTimestamp)) {
+ lastModifiedTimestamp = instant;
+ }
- metadata.set(ONE_NOTE_PREFIX + "lastModifiedTimestamp",
++ metadata.set(OneNote.LAST_MODIFIED_TIMESTAMP,
+ String.valueOf(lastModifiedTimestamp.toEpochMilli()));
+ } else if (oneNotePropertyEnum == OneNotePropertyEnum.CreationTimeStamp) {
+ // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not
+ // 1970
+ long scalar = getScalar(property);
+ long creationTs = scalar + TIME32_EPOCH_DIFF_1980;
+ if (creationTs < creationTimestamp) {
+ creationTimestamp = creationTs;
+ }
- metadata.set(ONE_NOTE_PREFIX + "creationTimestamp", String.valueOf(creationTimestamp));
++ metadata.set(OneNote.CREATION_TIMESTAMP, String.valueOf(creationTimestamp));
+ } else if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTime) {
+ // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not
+ // 1970
+ long scalar = getScalar(property);
+ long lastMod = scalar + TIME32_EPOCH_DIFF_1980;
+ if (lastMod > lastModified) {
+ lastModified = lastMod;
+ }
+ metadata.set(TikaCoreProperties.MODIFIED, String.valueOf(lastModified));
+ } else if (oneNotePropertyEnum == OneNotePropertyEnum.Author) {
+ String author = new String(((PrtFourBytesOfLengthFollowedByData) property).data,
+ StandardCharsets.UTF_8);
+ if (authorRole == AuthorRole.MOST_RECENT) {
+ mostRecentAuthors.add(author);
+ } else if (authorRole == AuthorRole.ORIGINAL) {
+ originalAuthors.add(author);
+ // the original authors are the creators of the content
+ authors.add(author);
+ } else {
+ authors.add(author);
+ }
+ } else if (propertyType == PropertyType.FourBytesOfLengthFollowedByData) {
+ boolean isBinary = propertyIsBinary(oneNotePropertyEnum);
+ PrtFourBytesOfLengthFollowedByData dataProperty =
+ (PrtFourBytesOfLengthFollowedByData) property;
+ if ((dataProperty.data.length & 1) == 0 &&
+ oneNotePropertyEnum != OneNotePropertyEnum.TextExtendedAscii && !isBinary) {
+ if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) {
+ xhtml.startElement(P);
+ xhtml.characters(new String(dataProperty.data, StandardCharsets.UTF_16LE));
+ xhtml.endElement(P);
+ }
+ } else if (oneNotePropertyEnum == OneNotePropertyEnum.TextExtendedAscii) {
+ xhtml.startElement(P);
+ xhtml.characters(new String(dataProperty.data, StandardCharsets.US_ASCII));
+ xhtml.endElement(P);
+ } else if (!isBinary) {
+ if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) {
+ xhtml.startElement(P);
+ xhtml.characters(new String(dataProperty.data, StandardCharsets.UTF_16LE));
+ xhtml.endElement(P);
+ }
+ } else {
+ if (oneNotePropertyEnum == OneNotePropertyEnum.RichEditTextUnicode) {
+ handleRichEditTextUnicode(dataProperty.data, xhtml);
+ } else {
+ //TODO -- these seem to be somewhat broken font files and other
+ //odds and ends...what are they and how should we process them?
+ //handleEmbedded(content.size());
+ }
+ }
+ }
+ }
+
+
+ /**
+ * Hands the binary data of an embedded object (e.g. an image or an attached file) to the
+ * embedded document extractor.
+ */
+ private void handleEmbedded(byte[] data, XHTMLContentHandler xhtml)
+ throws SAXException, IOException {
+ if (data == null || data.length == 0 || embeddedDocumentExtractor == null) {
+ return;
+ }
+ Metadata embeddedMetadata = Metadata.newInstance(this.parseContext);
+ AttributesImpl attributes = new AttributesImpl();
+ attributes.addAttribute("", "class", "class", "CDATA", "embedded");
+ xhtml.startElement("div", attributes);
+ xhtml.endElement("div");
+ try (TikaInputStream tis = TikaInputStream.get(data)) {
+ if (embeddedDocumentExtractor.shouldParseEmbedded(embeddedMetadata)) {
+ embeddedDocumentExtractor.parseEmbedded(tis, new EmbeddedContentHandler(xhtml),
+ embeddedMetadata, this.parseContext, false);
+ }
+ } catch (IOException e) {
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata);
+ }
+ }
private void handleRichEditTextUnicode(byte[] arr, XHTMLContentHandler xhtml)
throws SAXException, IOException, TikaException {
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java
index a44a9f75ed,25790ead8b..30ce905c67
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java
@@@ -69,10 -55,8 +69,11 @@@ public class MSOneStoreParser
private List<DataElement> objectGroupDataElements;
// The DataElements of Object BLOB
private List<DataElement> objectBlOBElements;
+ // The DataElements of Object BLOB, keyed by their data element extended GUID
+ private Map<ExGuid, DataElement> objectBlOBElementsById;
public MSOneStorePackage parse(DataElementPackage dataElementPackage) throws IOException {
++ storageIndexHashTab.clear();
MSOneStorePackage msOneStorePackage = new MSOneStorePackage();
storageIndexDataElements = dataElementPackage.dataElements.stream()
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java
index 6de0064661,afd2f6c067..dfa3f05d8f
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java
@@@ -83,6 -73,6 +83,9 @@@ public class RevisionStoreObjectGroup
for (int i = 0; i <
dataObject.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList.size();
i++) {
++ if (i >= dataObject.objectGroupData.objectGroupObjectDataBLOBReferenceList.size()) {
++ throw new IOException("Missing BLOB reference for object declaration " + i);
++ }
ObjectGroupObjectBLOBDataDeclaration objectGroupObjectBLOBDataDeclaration =
dataObject.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList.get(
i);
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
index 841415bb98,0000000000..e164455dab
mode 100644,000000..100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
@@@ -1,293 -1,0 +1,292 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.onenote.fsshttpb;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.ArrayNumber;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.IProperty;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.NoData;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtArrayOfPropertyValues;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtFourBytesOfLengthFollowedByData;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectData;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySet;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySetObject;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestRootDeclare;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreCell;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObject;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObjectGroup;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellID;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGuid;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyID;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyType;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectPropSet;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil;
+import org.apache.tika.sax.ToTextContentHandler;
+import org.apache.tika.sax.XHTMLContentHandler;
+
+public class MSOneStorePackageTest {
+
+ @Test
+ public void testPagesFollowSectionOrderAndDropOlderCellVersions() throws Exception {
+ ExGuid sectionRootId = id(1);
+ CellID pageOne = cell(10, 100);
+ CellID pageTwo = cell(20, 200);
+ CellID oldPageOne = cell(11, 100);
+ RevisionStoreCell pageTwoCell = cellWithText(pageTwo, "page two");
+ RevisionStoreCell oldPageOneCell = cellWithText(oldPageOne, "old page one");
+ RevisionStoreCell pageOneCell = cellWithText(pageOne, "page one");
+ RevisionStoreCell unrelatedCell = cellWithText(cell(30, 300), "unrelated");
+
+ RevisionStoreObject sectionRoot = object(sectionRootId,
+ propertySet(new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78,
+ new NoData()),
+ new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D79,
+ new NoData())),
+ Collections.emptyList(), Arrays.asList(pageOne, pageTwo));
+ RevisionStoreCell section = new RevisionStoreCell();
+ section.objectGroups.add(group(sectionRoot));
+ RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare();
+ rootDeclare.objectExGuid = sectionRootId;
+ section.rootDeclares.add(rootDeclare);
+ RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare();
+ missingRoot.objectExGuid = id(1000);
+ section.rootDeclares.add(missingRoot);
+
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.dataRootCell = section;
+ pkg.cells.addAll(Arrays.asList(pageTwoCell, oldPageOneCell, pageOneCell, unrelatedCell));
+
+ String text = walk(pkg);
+ assertTrue(text.indexOf("page one") < text.indexOf("page two"));
- assertTrue(text.contains("----------------------------------------"));
+ assertFalse(text.contains("old page one"));
+ assertTrue(text.contains("unrelated"));
+ }
+
+ @Test
+ public void testUnresolvedRootsFallBackToAllObjects() throws Exception {
+ RevisionStoreCell cell = cellWithText(cell(1, 1), "fallback content");
+ cell.rootDeclares.clear();
+ RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare();
+ missingRoot.objectExGuid = id(999);
+ cell.rootDeclares.add(missingRoot);
+
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ assertTrue(walk(pkg).contains("fallback content"));
+ }
+
+ @Test
+ public void testPrimaryPictureSuppressesDerivedPicture() throws Exception {
+ ExGuid pictureID = id(40);
+ ExGuid webPictureID = id(41);
+ RevisionStoreObject root = object(id(42), propertySet(
+ new PropertySpec(PropertyType.ObjectID, 0x20001C3F, new NoData()),
+ new PropertySpec(PropertyType.ObjectID, 0x200034C8, new NoData())),
+ Arrays.asList(pictureID, webPictureID), Collections.emptyList());
+ RevisionStoreObject picture = object(pictureID, propertySet(),
+ Collections.emptyList(), Collections.emptyList());
+ picture.propertySet.objectSpaceObjectPropSet.body = null;
+ RevisionStoreObject webPicture = object(webPictureID, propertySet(
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("derived picture"))),
+ Collections.emptyList(), Collections.emptyList());
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(root, picture, webPicture));
+ RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare();
+ rootDeclare.objectExGuid = root.objectID;
+ cell.rootDeclares.add(rootDeclare);
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+
+ assertFalse(walk(pkg).contains("derived picture"));
+ }
+
+ @Test
+ public void testNestedPropertySetsAndMissingReferencesAreTraversedSafely() throws Exception {
+ ExGuid childId = id(2);
+ PropertySet nested = propertySet(new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("nested text")));
+ PrtArrayOfPropertyValues array = new PrtArrayOfPropertyValues();
+ array.data = new PropertySet[]{propertySet(new PropertySpec(
+ PropertyType.FourBytesOfLengthFollowedByData, 0x1C003498, text("array text")))};
+ RevisionStoreObject root = object(id(1), propertySet(
+ new PropertySpec(PropertyType.PropertySet, 0, nested),
+ new PropertySpec(PropertyType.ArrayOfPropertyValues, 0, array),
+ new PropertySpec(PropertyType.ArrayOfObjectIDs, 0x24001D5F,
+ arrayNumber(2)),
+ new PropertySpec(PropertyType.ObjectID, 0x24001D5F, new NoData()),
+ new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78, new NoData()),
+ new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D79, new NoData()),
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001DD7, bytes((byte) 'u', (byte) 0, (byte) 1)),
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001C22, bytes((byte) 'h', (byte) 0, (byte) 'i', (byte) 0,
+ (byte) 0, (byte) 0))),
+ Collections.singletonList(childId), Collections.singletonList(cell(50, 51)));
+ RevisionStoreObject child = object(childId, propertySet(
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("child text"))),
+ Collections.emptyList(), Collections.emptyList());
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(root, child));
+ RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare();
+ rootDeclare.objectExGuid = root.objectID;
+ cell.rootDeclares.add(rootDeclare);
+
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+ String text = walk(pkg);
+ assertTrue(text.contains("nested text"));
+ assertTrue(text.contains("array text"));
+ assertTrue(text.contains("child text"));
+ assertTrue(text.contains("u"));
+ assertTrue(text.contains("hi"));
+ }
+
+ private static String walk(MSOneStorePackage pkg) throws Exception {
+ Metadata metadata = new Metadata();
+ StringWriter writer = new StringWriter();
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(
+ new ToTextContentHandler(writer), metadata, new ParseContext());
+ xhtml.startDocument();
+ pkg.walkTree(new OneNoteTreeWalkerOptions(), metadata, xhtml);
+ xhtml.endDocument();
+ return writer.toString();
+ }
+
+ private static RevisionStoreCell cellWithText(CellID cellID, String value) throws Exception {
+ RevisionStoreObject object = object(id(cellID.extendGUID1.hashCode()),
+ propertySet(new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text(value))), Collections.emptyList(), Collections.emptyList());
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.cellID = cellID;
+ cell.objectGroups.add(group(object));
+ RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare();
+ rootDeclare.objectExGuid = object.objectID;
+ cell.rootDeclares.add(rootDeclare);
+ return cell;
+ }
+
+ private static RevisionStoreObject object(ExGuid objectID, PropertySet body,
+ List<ExGuid> references, List<CellID> spaces)
+ throws Exception {
+ RevisionStoreObject object = new RevisionStoreObject();
+ object.objectID = objectID;
+ PropertySetObject propertySetObject = new PropertySetObject(null, emptyObjectData());
+ ObjectSpaceObjectPropSet propSet = new ObjectSpaceObjectPropSet();
+ propSet.body = body;
+ propertySetObject.objectSpaceObjectPropSet = propSet;
+ object.propertySet = propertySetObject;
+ if (!references.isEmpty()) {
+ object.referencedObjectID = new org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGUIDArray();
+ object.referencedObjectID.content = references;
+ }
+ if (!spaces.isEmpty()) {
+ object.referencedObjectSpacesID = new org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellIDArray();
+ object.referencedObjectSpacesID.content = spaces;
+ }
+ return object;
+ }
+
+ private static RevisionStoreObjectGroup group(RevisionStoreObject... objects) {
+ RevisionStoreObjectGroup group = new RevisionStoreObjectGroup(id(500));
+ group.objects.addAll(Arrays.asList(objects));
+ return group;
+ }
+
+ private static PropertySet propertySet(PropertySpec... specs) {
+ PropertySet set = new PropertySet();
+ set.cProperties = specs.length;
+ set.rgPrids = new PropertyID[specs.length];
+ set.rgData = new ArrayList<>();
+ for (int i = 0; i < specs.length; i++) {
+ set.rgPrids[i] = propertyID(specs[i].type, specs[i].value);
+ set.rgData.add(specs[i].property);
+ }
+ return set;
+ }
+
+ private static PropertyID propertyID(PropertyType type, int value) {
+ PropertyID id = new PropertyID();
+ id.type = type.getIntVal();
+ id.value = value;
+ return id;
+ }
+
+ private static PrtFourBytesOfLengthFollowedByData bytes(byte... value) {
+ PrtFourBytesOfLengthFollowedByData data = new PrtFourBytesOfLengthFollowedByData();
+ data.data = value;
+ data.cb = data.data.length;
+ return data;
+ }
+
+ private static PrtFourBytesOfLengthFollowedByData text(String value) {
+ PrtFourBytesOfLengthFollowedByData data = new PrtFourBytesOfLengthFollowedByData();
+ data.data = value.getBytes(StandardCharsets.US_ASCII);
+ data.cb = data.data.length;
+ return data;
+ }
+
+ private static ArrayNumber arrayNumber(int number) {
+ ArrayNumber array = new ArrayNumber();
+ array.number = number;
+ return array;
+ }
+
+ private static ObjectGroupObjectData emptyObjectData() {
+ ObjectGroupObjectData data = new ObjectGroupObjectData();
+ data.data.content.addAll(ByteUtil.toListOfByte(new byte[]{0, 0, 0, (byte) 0x80,
+ 0, 0, 0, 0}));
+ return data;
+ }
+
+ private static CellID cell(int first, int second) {
+ return new CellID(id(first), id(second));
+ }
+
+ private static ExGuid id(int value) {
+ return new ExGuid(value, UUID.nameUUIDFromBytes(("id-" + value).getBytes(StandardCharsets.UTF_8)));
+ }
+
+ private static final class PropertySpec {
+ private final PropertyType type;
+ private final int value;
+ private final IProperty property;
+
+ private PropertySpec(PropertyType type, int value, IProperty property) {
+ this.type = type;
+ this.value = value;
+ this.property = property;
+ }
+ }
+}