You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ni...@apache.org on 2018/05/18 14:35:35 UTC
[tika] branch master updated (5f05b51 -> 12693ea)
This is an automated email from the ASF dual-hosted git repository.
nick pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git.
from 5f05b51 TIKA-2644 - refactor recursiveparserwrapper api
new a1e42a0 TIKA-2479 Option to request missing rows where possible in Excel-like formats
new b1b035e TIKA-2479 Output missing left/mid cells in XLSX and XLSB, and optionally also missing rows
new b01b059 Updated Columnar output from SAS with better formats
new 6fa1105 Formatted columns in the columnar test Excel files
new 348b87e TIKA-2479 Update XLS missing cell/row handling to match XLSX and XLSB, add unit test for missing rows, and enable the Columnar tests for the Excel formats
new 9673fbd Changelog update
new 12693ea Add the other jackcess jar to the bundle
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.txt | 5 +++
tika-bundle/pom.xml | 1 +
.../tika/parser/microsoft/ExcelExtractor.java | 26 +++++++-------
.../tika/parser/microsoft/OfficeParserConfig.java | 16 ++++++++-
.../ooxml/XSSFBExcelExtractorDecorator.java | 2 +-
.../ooxml/XSSFExcelExtractorDecorator.java | 35 +++++++++++++++---
.../org/apache/tika/parser/TabularFormatsTest.java | 40 ++++++++++++---------
.../tika/parser/microsoft/ExcelParserTest.java | 25 ++++++++++++-
.../apache/tika/parser/sas/SAS7BDATParserTest.java | 8 ++---
.../test-documents/test-columnar.sas7bdat | Bin 17408 -> 131072 bytes
.../resources/test-documents/test-columnar.xls | Bin 6656 -> 32768 bytes
.../resources/test-documents/test-columnar.xlsb | Bin 0 -> 9691 bytes
.../resources/test-documents/test-columnar.xlsx | Bin 4941 -> 10556 bytes
.../src/test/resources/test-documents/testSAS2.sas | 3 ++
14 files changed, 120 insertions(+), 41 deletions(-)
create mode 100644 tika-parsers/src/test/resources/test-documents/test-columnar.xlsb
--
To stop receiving notification emails like this one, please contact
nick@apache.org.
[tika] 06/07: Changelog update
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nick pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 9673fbdbba8feebb72fee569074e94b0868a89df
Author: Nick Burch <ni...@gagravarr.org>
AuthorDate: Fri May 18 15:17:56 2018 +0100
Changelog update
---
CHANGES.txt | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/CHANGES.txt b/CHANGES.txt
index 38f1973..0ffc5de 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -81,6 +81,11 @@ Release 2.0.0 - ???
* Mime magic for ACES Images (TIKA-2628) and DPX Images (TIKA-2629)
+ * For sparse XLSX and XLSB files, always output missing cells to
+ the left of filled ones (matching XLS), and optionally output
+ missing rows on all 3 formats if requested via the
+ OfficeParserContext (TIKA-2479)
+
Release 1.17 - 12/8/2017
***NOTE: THIS IS THE LAST VERSION OF TIKA THAT WILL RUN
--
To stop receiving notification emails like this one, please contact
nick@apache.org.
[tika] 02/07: TIKA-2479 Output missing left/mid cells in XLSX and
XLSB, and optionally also missing rows
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nick pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit b1b035e6bbcff0db24e133b682ac79916f92f599
Author: Nick Burch <ni...@gagravarr.org>
AuthorDate: Thu May 17 23:07:04 2018 +0100
TIKA-2479 Output missing left/mid cells in XLSX and XLSB, and optionally also missing rows
---
.../ooxml/XSSFBExcelExtractorDecorator.java | 2 +-
.../ooxml/XSSFExcelExtractorDecorator.java | 35 ++++++++++++++++++----
.../org/apache/tika/parser/TabularFormatsTest.java | 11 ++++++-
3 files changed, 41 insertions(+), 7 deletions(-)
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFBExcelExtractorDecorator.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFBExcelExtractorDecorator.java
index dcde62b..33dbb7e 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFBExcelExtractorDecorator.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFBExcelExtractorDecorator.java
@@ -117,7 +117,7 @@ public class XSSFBExcelExtractorDecorator extends XSSFExcelExtractorDecorator {
addDrawingHyperLinks(sheetPart);
sheetParts.add(sheetPart);
- SheetTextAsHTML sheetExtractor = new SheetTextAsHTML(config.getIncludeHeadersAndFooters(), xhtml);
+ SheetTextAsHTML sheetExtractor = new SheetTextAsHTML(config, xhtml);
XSSFBCommentsTable comments = iter.getXSSFBSheetComments();
// Start, and output the sheet name
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java
index 9a2b017..7e1a7cd 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
-import org.apache.poi.POIXMLDocument;
import org.apache.poi.POIXMLTextExtractor;
import org.apache.poi.hssf.extractor.ExcelExtractor;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
@@ -39,6 +38,7 @@ import org.apache.poi.openxml4j.opc.PackagingURIHelper;
import org.apache.poi.openxml4j.opc.TargetMode;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.HeaderFooter;
+import org.apache.poi.ss.util.CellReference;
import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;
import org.apache.poi.xssf.eventusermodel.XSSFReader;
import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler;
@@ -56,6 +56,7 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.microsoft.OfficeParserConfig;
import org.apache.tika.parser.microsoft.TikaExcelDataFormatter;
import org.apache.tika.sax.OfflineContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
@@ -144,8 +145,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
}
while (iter.hasNext()) {
-
- SheetTextAsHTML sheetExtractor = new SheetTextAsHTML(config.getIncludeHeadersAndFooters(), xhtml);
+ SheetTextAsHTML sheetExtractor = new SheetTextAsHTML(config, xhtml);
PackagePart sheetPart = null;
try (InputStream stream = iter.next()) {
sheetPart = iter.getSheetPart();
@@ -397,11 +397,15 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
protected static class SheetTextAsHTML implements SheetContentsHandler {
private XHTMLContentHandler xhtml;
private final boolean includeHeadersFooters;
+ private final boolean includeMissingRows;
protected List<String> headers;
protected List<String> footers;
+ private int lastSeenRow = -1;
+ private int lastSeenCol = -1;
- protected SheetTextAsHTML(boolean includeHeaderFooters, XHTMLContentHandler xhtml) {
- this.includeHeadersFooters = includeHeaderFooters;
+ protected SheetTextAsHTML(OfficeParserConfig config, XHTMLContentHandler xhtml) {
+ this.includeHeadersFooters = config.getIncludeHeadersAndFooters();
+ this.includeMissingRows = config.getIncludeMissingRows();
this.xhtml = xhtml;
headers = new ArrayList<String>();
footers = new ArrayList<String>();
@@ -409,7 +413,19 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
public void startRow(int rowNum) {
try {
+ // Missing rows, if desired, with a single empty row
+ if (includeMissingRows && rowNum > (lastSeenRow+1)) {
+ for (int rn=lastSeenRow+1; rn<rowNum; rn++) {
+ xhtml.startElement("tr");
+ xhtml.startElement("td");
+ xhtml.endElement("td");
+ xhtml.endElement("tr");
+ }
+ }
+
+ // Start the new row
xhtml.startElement("tr");
+ lastSeenCol = -1;
} catch (SAXException e) {
}
}
@@ -423,6 +439,15 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
public void cell(String cellRef, String formattedValue, XSSFComment comment) {
try {
+ // Handle any missing cells
+ int colNum = (new CellReference(cellRef)).getCol();
+ for (int cn=lastSeenCol+1; cn<colNum; cn++) {
+ xhtml.startElement("td");
+ xhtml.endElement("td");
+ }
+ lastSeenCol = colNum;
+
+ // Start this cell
xhtml.startElement("td");
// Main cell contents
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/TabularFormatsTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/TabularFormatsTest.java
index ea326bd..41139e2 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/TabularFormatsTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/TabularFormatsTest.java
@@ -234,9 +234,18 @@ public class TabularFormatsTest extends TikaTest {
XMLResult result = getXML("test-columnar.xlsx");
String xml = result.xml;
assertHeaders(xml, false, true, false);
- // TODO Correctly handle empty cells then enable this test
+ // TODO Fix formatting in export then enable this test
//assertContents(xml, true, false);
}
+ // Get a test XLSB file, then enable this unit test
+// @Test
+// public void testXLSB() throws Exception {
+// XMLResult result = getXML("test-columnar.xlsb");
+// String xml = result.xml;
+// assertHeaders(xml, false, true, false);
+// assertContents(xml, true, false);
+// }
+
// TODO Fix the ODS test - currently failing with
// org.xml.sax.SAXException: Namespace http://www.w3.org/1999/xhtml not declared
// @Test
--
To stop receiving notification emails like this one, please contact
nick@apache.org.
[tika] 04/07: Formatted columns in the columnar test Excel files
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nick pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 6fa1105e0669ffeec5c3cf0d1db247a8c16f3bc5
Author: Nick Burch <ni...@gagravarr.org>
AuthorDate: Fri May 18 15:13:43 2018 +0100
Formatted columns in the columnar test Excel files
---
.../test/resources/test-documents/test-columnar.xls | Bin 66048 -> 32768 bytes
.../resources/test-documents/test-columnar.xlsb | Bin 0 -> 9691 bytes
.../resources/test-documents/test-columnar.xlsx | Bin 6603 -> 10556 bytes
.../src/test/resources/test-documents/testSAS2.sas | 3 +++
4 files changed, 3 insertions(+)
diff --git a/tika-parsers/src/test/resources/test-documents/test-columnar.xls b/tika-parsers/src/test/resources/test-documents/test-columnar.xls
index cc45372..3f1009c 100644
Binary files a/tika-parsers/src/test/resources/test-documents/test-columnar.xls and b/tika-parsers/src/test/resources/test-documents/test-columnar.xls differ
diff --git a/tika-parsers/src/test/resources/test-documents/test-columnar.xlsb b/tika-parsers/src/test/resources/test-documents/test-columnar.xlsb
new file mode 100644
index 0000000..0ce5139
Binary files /dev/null and b/tika-parsers/src/test/resources/test-documents/test-columnar.xlsb differ
diff --git a/tika-parsers/src/test/resources/test-documents/test-columnar.xlsx b/tika-parsers/src/test/resources/test-documents/test-columnar.xlsx
index 22483f1..f1f4dc4 100644
Binary files a/tika-parsers/src/test/resources/test-documents/test-columnar.xlsx and b/tika-parsers/src/test/resources/test-documents/test-columnar.xlsx differ
diff --git a/tika-parsers/src/test/resources/test-documents/testSAS2.sas b/tika-parsers/src/test/resources/test-documents/testSAS2.sas
index 96a9121..df52b1a 100644
--- a/tika-parsers/src/test/resources/test-documents/testSAS2.sas
+++ b/tika-parsers/src/test/resources/test-documents/testSAS2.sas
@@ -57,6 +57,9 @@ proc export data=testing label
putnames=yes;
run;
+/* Due to SAS Limitations, you will need to manually */
+/* style the % and Date/Datetime columns in Excel */
+/* You will also need to save-as XLSB to generate that */
proc export data=testing label
outfile="&outpath./testing.xls"
dbms=XLS;
--
To stop receiving notification emails like this one, please contact
nick@apache.org.
[tika] 01/07: TIKA-2479 Option to request missing rows where
possible in Excel-like formats
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nick pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit a1e42a0659ba33e90cb1bba0a0a10eeb97d4fac7
Author: Nick Burch <ni...@gagravarr.org>
AuthorDate: Thu May 17 22:15:34 2018 +0100
TIKA-2479 Option to request missing rows where possible in Excel-like formats
---
.../apache/tika/parser/microsoft/OfficeParserConfig.java | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java
index 34b865e..5d34b2e 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java
@@ -29,6 +29,7 @@ public class OfficeParserConfig implements Serializable {
private boolean includeMoveFromContent = false;
private boolean includeShapeBasedContent = true;
private boolean includeHeadersAndFooters = true;
+ private boolean includeMissingRows = false;
private boolean concatenatePhoneticRuns = true;
private boolean useSAXDocxExtractor = false;
@@ -188,10 +189,23 @@ public class OfficeParserConfig implements Serializable {
this.extractAllAlternativesFromMSG = extractAllAlternativesFromMSG;
}
-
public boolean getExtractAllAlternativesFromMSG() {
return extractAllAlternativesFromMSG;
}
+
+ /**
+ * For table-like formats, and tables within other formats, should
+ * missing rows in sparse tables be output where detected?
+ * The default is to only output rows defined within the file, which
+ * avoid lots of blank lines, but means layout isn't preserved.
+ */
+ public void setIncludeMissingRows(boolean includeMissingRows) {
+ this.includeMissingRows = includeMissingRows;
+ }
+
+ public boolean getIncludeMissingRows() {
+ return includeMissingRows;
+ }
}
--
To stop receiving notification emails like this one, please contact
nick@apache.org.
[tika] 07/07: Add the other jackcess jar to the bundle
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nick pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 12693ea18f1a05894272aa3a9293d41215f63c06
Author: Nick Burch <ni...@gagravarr.org>
AuthorDate: Fri May 18 15:35:06 2018 +0100
Add the other jackcess jar to the bundle
---
tika-bundle/pom.xml | 1 +
1 file changed, 1 insertion(+)
diff --git a/tika-bundle/pom.xml b/tika-bundle/pom.xml
index 2b500d7..fa13e21 100644
--- a/tika-bundle/pom.xml
+++ b/tika-bundle/pom.xml
@@ -170,6 +170,7 @@
curvesapi|
xmlbeans|
jackcess|
+ jackcess-encrypt|
commons-lang|
tagsoup|
asm|
--
To stop receiving notification emails like this one, please contact
nick@apache.org.
[tika] 03/07: Updated Columnar output from SAS with better formats
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nick pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit b01b059331f198d3829b111002cf03cbcaf1bab3
Author: Nick Burch <ni...@gagravarr.org>
AuthorDate: Fri May 18 11:43:47 2018 +0100
Updated Columnar output from SAS with better formats
---
.../apache/tika/parser/sas/SAS7BDATParserTest.java | 8 ++++----
.../test-documents/test-columnar.sas7bdat | Bin 17408 -> 131072 bytes
.../resources/test-documents/test-columnar.xls | Bin 6656 -> 66048 bytes
.../resources/test-documents/test-columnar.xlsx | Bin 4941 -> 6603 bytes
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/sas/SAS7BDATParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/sas/SAS7BDATParserTest.java
index 610ffc3..00a2aaa 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/sas/SAS7BDATParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/sas/SAS7BDATParserTest.java
@@ -89,15 +89,15 @@ public class SAS7BDATParserTest extends TikaTest {
assertEquals("application/x-sas-data", metadata.get(Metadata.CONTENT_TYPE));
assertEquals("TESTING", metadata.get(TikaCoreProperties.TITLE));
- assertEquals("2018-05-09T17:59:33Z", metadata.get(TikaCoreProperties.CREATED));
- assertEquals("2018-05-09T17:59:33Z", metadata.get(TikaCoreProperties.MODIFIED));
+ assertEquals("2018-05-18T11:38:30Z", metadata.get(TikaCoreProperties.CREATED));
+ assertEquals("2018-05-18T11:38:30Z", metadata.get(TikaCoreProperties.MODIFIED));
assertEquals("1", metadata.get(PagedText.N_PAGES));
assertEquals("8", metadata.get(Database.COLUMN_COUNT));
assertEquals("11", metadata.get(Database.ROW_COUNT));
assertEquals("windows-1252", metadata.get(HttpHeaders.CONTENT_ENCODING));
- assertEquals("W32_7PRO", metadata.get(OfficeOpenXMLExtended.APPLICATION));
- assertEquals("9.0301M2", metadata.get(OfficeOpenXMLExtended.APP_VERSION));
+ assertEquals("X64_7PRO", metadata.get(OfficeOpenXMLExtended.APPLICATION));
+ assertEquals("9.0401M5", metadata.get(OfficeOpenXMLExtended.APP_VERSION));
assertEquals("32", metadata.get(MachineMetadata.ARCHITECTURE_BITS));
assertEquals("Little", metadata.get(MachineMetadata.ENDIAN));
assertEquals(Arrays.asList("Record Number","Square of the Record Number",
diff --git a/tika-parsers/src/test/resources/test-documents/test-columnar.sas7bdat b/tika-parsers/src/test/resources/test-documents/test-columnar.sas7bdat
index 33ee412..f6cab63 100644
Binary files a/tika-parsers/src/test/resources/test-documents/test-columnar.sas7bdat and b/tika-parsers/src/test/resources/test-documents/test-columnar.sas7bdat differ
diff --git a/tika-parsers/src/test/resources/test-documents/test-columnar.xls b/tika-parsers/src/test/resources/test-documents/test-columnar.xls
index 1d7b2cf..cc45372 100644
Binary files a/tika-parsers/src/test/resources/test-documents/test-columnar.xls and b/tika-parsers/src/test/resources/test-documents/test-columnar.xls differ
diff --git a/tika-parsers/src/test/resources/test-documents/test-columnar.xlsx b/tika-parsers/src/test/resources/test-documents/test-columnar.xlsx
index 58ffd47..22483f1 100644
Binary files a/tika-parsers/src/test/resources/test-documents/test-columnar.xlsx and b/tika-parsers/src/test/resources/test-documents/test-columnar.xlsx differ
--
To stop receiving notification emails like this one, please contact
nick@apache.org.
[tika] 05/07: TIKA-2479 Update XLS missing cell/row handling to
match XLSX and XLSB, add unit test for missing rows,
and enable the Columnar tests for the Excel formats
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nick pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 348b87e7f41b79ff115e17d9c91d2dad63a57c15
Author: Nick Burch <ni...@gagravarr.org>
AuthorDate: Fri May 18 15:15:32 2018 +0100
TIKA-2479 Update XLS missing cell/row handling to match XLSX and XLSB, add unit test for missing rows, and enable the Columnar tests for the Excel formats
---
.../tika/parser/microsoft/ExcelExtractor.java | 26 ++++++------
.../org/apache/tika/parser/TabularFormatsTest.java | 47 ++++++++++------------
.../tika/parser/microsoft/ExcelParserTest.java | 25 +++++++++++-
3 files changed, 60 insertions(+), 38 deletions(-)
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ExcelExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ExcelExtractor.java
index 0dc33ee..ff5971a 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ExcelExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ExcelExtractor.java
@@ -16,7 +16,7 @@
*/
package org.apache.tika.parser.microsoft;
-import java.awt.*;
+import java.awt.Point;
import java.io.IOException;
import java.text.NumberFormat;
import java.util.ArrayList;
@@ -42,7 +42,6 @@ import org.apache.poi.hssf.record.CountryRecord;
import org.apache.poi.hssf.record.DateWindow1904Record;
import org.apache.poi.hssf.record.DrawingGroupRecord;
import org.apache.poi.hssf.record.EOFRecord;
-import org.apache.poi.hssf.record.ExtSSTRecord;
import org.apache.poi.hssf.record.ExtendedFormatRecord;
import org.apache.poi.hssf.record.FooterRecord;
import org.apache.poi.hssf.record.FormatRecord;
@@ -281,7 +280,6 @@ public class ExcelExtractor extends AbstractPOIFSExtractor {
public void processFile(DirectoryNode root, boolean listenForAllRecords)
throws IOException, SAXException, TikaException {
-
// Set up listener and register the records we want to process
HSSFRequest hssfRequest = new HSSFRequest();
if (listenForAllRecords) {
@@ -494,15 +492,14 @@ public class ExcelExtractor extends AbstractPOIFSExtractor {
HeaderRecord headerRecord = (HeaderRecord) record;
addTextCell(record, headerRecord.getText());
}
- break;
+ break;
case FooterRecord.sid:
if (extractor.officeParserConfig.getIncludeHeadersAndFooters()) {
FooterRecord footerRecord = (FooterRecord) record;
addTextCell(record, footerRecord.getText());
}
- break;
-
+ break;
}
previousSid = record.getSid();
@@ -599,12 +596,17 @@ public class ExcelExtractor extends AbstractPOIFSExtractor {
handler.startElement("tr");
handler.startElement("td");
for (Map.Entry<Point, Cell> entry : currentSheet.entrySet()) {
- while (currentRow < entry.getKey().y) {
- handler.endElement("td");
- handler.endElement("tr");
- handler.startElement("tr");
- handler.startElement("td");
- currentRow++;
+ if (currentRow != entry.getKey().y) {
+ // We've moved onto a new row, possibly skipping some
+ do {
+ handler.endElement("td");
+ handler.endElement("tr");
+ handler.startElement("tr");
+ handler.startElement("td");
+ currentRow++;
+ } while (officeParserConfig.getIncludeMissingRows() &&
+ currentRow < entry.getKey().y);
+ currentRow = entry.getKey().y;
currentColumn = 0;
}
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/TabularFormatsTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/TabularFormatsTest.java
index 41139e2..4a52118 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/TabularFormatsTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/TabularFormatsTest.java
@@ -64,8 +64,8 @@ public class TabularFormatsTest extends TikaTest {
"87.5%","88.9%","90.0%"
},
new Pattern[] {
- Pattern.compile("01-(01|JAN|Jan)-(60|1960)"),
- Pattern.compile("02-01-1960"),
+ Pattern.compile("0?1-01-1960"),
+ Pattern.compile("0?2-01-1960"),
Pattern.compile("17-01-1960"),
Pattern.compile("22-03-1960"),
Pattern.compile("13-09-1960"),
@@ -77,17 +77,17 @@ public class TabularFormatsTest extends TikaTest {
Pattern.compile("19-05-1987"),
},
new Pattern[] {
- Pattern.compile("01(JAN|Jan)(60|1960):00:00:01(.00)?"),
- Pattern.compile("01(JAN|Jan)(60|1960):00:00:10(.00)?"),
- Pattern.compile("01(JAN|Jan)(60|1960):00:01:40(.00)?"),
- Pattern.compile("01(JAN|Jan)(60|1960):00:16:40(.00)?"),
- Pattern.compile("01(JAN|Jan)(60|1960):02:46:40(.00)?"),
- Pattern.compile("02(JAN|Jan)(60|1960):03:46:40(.00)?"),
- Pattern.compile("12(JAN|Jan)(60|1960):13:46:40(.00)?"),
- Pattern.compile("25(APR|Apr)(60|1960):17:46:40(.00)?"),
- Pattern.compile("03(MAR|Mar)(63|1963):09:46:40(.00)?"),
- Pattern.compile("09(SEP|Sep)(91|1991):01:46:40(.00)?"),
- Pattern.compile("19(NOV|Nov)(76|2276):17:46:40(.00)?")
+ Pattern.compile("01(JAN|Jan)(60|1960)[:\\s]00:00:01(.00)?"),
+ Pattern.compile("01(JAN|Jan)(60|1960)[:\\s]00:00:10(.00)?"),
+ Pattern.compile("01(JAN|Jan)(60|1960)[:\\s]00:01:40(.00)?"),
+ Pattern.compile("01(JAN|Jan)(60|1960)[:\\s]00:16:40(.00)?"),
+ Pattern.compile("01(JAN|Jan)(60|1960)[:\\s]02:46:40(.00)?"),
+ Pattern.compile("02(JAN|Jan)(60|1960)[:\\s]03:46:40(.00)?"),
+ Pattern.compile("12(JAN|Jan)(60|1960)[:\\s]13:46:40(.00)?"),
+ Pattern.compile("25(APR|Apr)(60|1960)[:\\s]17:46:40(.00)?"),
+ Pattern.compile("03(MAR|Mar)(63|1963)[:\\s]09:46:40(.00)?"),
+ Pattern.compile("09(SEP|Sep)(91|1991)[:\\s]01:46:40(.00)?"),
+ Pattern.compile("19(NOV|Nov)(76|2276)[:\\s]17:46:40(.00)?")
},
new Pattern[] {
Pattern.compile("0?0:00:01(.\\d\\d)?"),
@@ -226,25 +226,22 @@ public class TabularFormatsTest extends TikaTest {
XMLResult result = getXML("test-columnar.xls");
String xml = result.xml;
assertHeaders(xml, false, true, false);
- // TODO Correctly handle empty cells then enable this test
- //assertContents(xml, true, false);
+ assertContents(xml, true, false);
}
@Test
public void testXLSX() throws Exception {
XMLResult result = getXML("test-columnar.xlsx");
String xml = result.xml;
assertHeaders(xml, false, true, false);
- // TODO Fix formatting in export then enable this test
- //assertContents(xml, true, false);
+ assertContents(xml, true, false);
+ }
+ @Test
+ public void testXLSB() throws Exception {
+ XMLResult result = getXML("test-columnar.xlsb");
+ String xml = result.xml;
+ assertHeaders(xml, false, true, false);
+ assertContents(xml, true, false);
}
- // Get a test XLSB file, then enable this unit test
-// @Test
-// public void testXLSB() throws Exception {
-// XMLResult result = getXML("test-columnar.xlsb");
-// String xml = result.xml;
-// assertHeaders(xml, false, true, false);
-// assertContents(xml, true, false);
-// }
// TODO Fix the ODS test - currently failing with
// org.xml.sax.SAXException: Namespace http://www.w3.org/1999/xhtml not declared
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ExcelParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ExcelParserTest.java
index 732c11c..6304402 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ExcelParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ExcelParserTest.java
@@ -20,7 +20,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.File;
import java.io.InputStream;
import java.text.DecimalFormatSymbols;
import java.util.List;
@@ -78,6 +77,30 @@ public class ExcelParserTest extends TikaTest {
assertNotContained("9.0", content);
assertContains("196", content);
assertNotContained("196.0", content);
+
+
+ // Won't include missing rows by default
+ assertContains("Numbers and their Squares\n\t\tNumber", content);
+ assertContains("\tSquare\n\t\t1", content);
+ }
+
+ // Request with missing rows
+ try (InputStream input = ExcelParserTest.class.getResourceAsStream(
+ "/test-documents/testEXCEL.xls")) {
+ OfficeParserConfig config = new OfficeParserConfig();
+ config.setIncludeMissingRows(true);
+
+ Metadata metadata = new Metadata();
+ ContentHandler handler = new BodyContentHandler();
+ ParseContext context = new ParseContext();
+ context.set(Locale.class, Locale.US);
+ context.set(OfficeParserConfig.class, config);
+ new OfficeParser().parse(input, handler, metadata, context);
+
+ // Will now have the missing rows, each with a single empty cell
+ String content = handler.toString();
+ assertContains("Numbers and their Squares\n\t\n\t\n\t\tNumber", content);
+ assertContains("\tSquare\n\t\n\t\t1", content);
}
}
--
To stop receiving notification emails like this one, please contact
nick@apache.org.