You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@poi.apache.org by ce...@apache.org on 2017/03/18 13:56:12 UTC
svn commit: r1787556 - in /poi/trunk/src/integrationtest/org/apache/poi: ./
stress/
Author: centic
Date: Sat Mar 18 13:56:11 2017
New Revision: 1787556
URL: http://svn.apache.org/viewvc?rev=1787556&view=rev
Log:
Adjust integration-test files slightly, more failure-output, make it a bit easier to test single-files, add a HeapDump class for investigating memory leaks, adjust XSSFFileHandler to free memory more quickly
Added:
poi/trunk/src/integrationtest/org/apache/poi/HeapDump.java
Modified:
poi/trunk/src/integrationtest/org/apache/poi/stress/AbstractFileHandler.java
poi/trunk/src/integrationtest/org/apache/poi/stress/FileHandler.java
poi/trunk/src/integrationtest/org/apache/poi/stress/HSLFFileHandler.java
poi/trunk/src/integrationtest/org/apache/poi/stress/HSMFFileHandler.java
poi/trunk/src/integrationtest/org/apache/poi/stress/HSSFFileHandler.java
poi/trunk/src/integrationtest/org/apache/poi/stress/OPCFileHandler.java
poi/trunk/src/integrationtest/org/apache/poi/stress/SlideShowHandler.java
poi/trunk/src/integrationtest/org/apache/poi/stress/XSLFFileHandler.java
poi/trunk/src/integrationtest/org/apache/poi/stress/XSSFFileHandler.java
Added: poi/trunk/src/integrationtest/org/apache/poi/HeapDump.java
URL: http://svn.apache.org/viewvc/poi/trunk/src/integrationtest/org/apache/poi/HeapDump.java?rev=1787556&view=auto
==============================================================================
--- poi/trunk/src/integrationtest/org/apache/poi/HeapDump.java (added)
+++ poi/trunk/src/integrationtest/org/apache/poi/HeapDump.java Sat Mar 18 13:56:11 2017
@@ -0,0 +1,62 @@
+/* ====================================================================
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+==================================================================== */
+package org.apache.poi;
+
+import com.sun.management.HotSpotDiagnosticMXBean;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+
+public class HeapDump {
+ // This is the name of the HotSpot Diagnostic MBean
+ private static final String HOTSPOT_BEAN_NAME =
+ "com.sun.management:type=HotSpotDiagnostic";
+
+ // field to store the hotspot diagnostic MBean
+ private static volatile HotSpotDiagnosticMXBean hotspotMBean;
+
+ /**
+ * Call this method from your application whenever you
+ * want to dump the heap snapshot into a file.
+ *
+ * @param fileName name of the heap dump file
+ * @param live flag that tells whether to dump
+ * only the live objects
+ */
+ public static void dumpHeap(String fileName, boolean live) throws IOException {
+ // initialize hotspot diagnostic MBean
+ initHotspotMBean();
+ hotspotMBean.dumpHeap(fileName, live);
+ }
+
+ // initialize the hotspot diagnostic MBean field
+ private static void initHotspotMBean() throws IOException {
+ if (hotspotMBean == null) {
+ synchronized (HeapDump.class) {
+ if (hotspotMBean == null) {
+ hotspotMBean = getHotspotMBean();
+ }
+ }
+ }
+ }
+
+ // get the hotspot diagnostic MBean from the platform MBean server
+ private static HotSpotDiagnosticMXBean getHotspotMBean() throws IOException {
+ return ManagementFactory.newPlatformMXBeanProxy(ManagementFactory.getPlatformMBeanServer(),
+ HOTSPOT_BEAN_NAME, HotSpotDiagnosticMXBean.class);
+ }
+}
Modified: poi/trunk/src/integrationtest/org/apache/poi/stress/AbstractFileHandler.java
URL: http://svn.apache.org/viewvc/poi/trunk/src/integrationtest/org/apache/poi/stress/AbstractFileHandler.java?rev=1787556&r1=1787555&r2=1787556&view=diff
==============================================================================
--- poi/trunk/src/integrationtest/org/apache/poi/stress/AbstractFileHandler.java (original)
+++ poi/trunk/src/integrationtest/org/apache/poi/stress/AbstractFileHandler.java Sat Mar 18 13:56:11 2017
@@ -81,9 +81,9 @@ public abstract class AbstractFileHandle
POITextExtractor extractor = ExtractorFactory.createExtractor(file);
try {
- assertNotNull(extractor);
+ assertNotNull("Should get a POITextExtractor but had none for file " + file, extractor);
- assertNotNull(extractor.getText());
+ assertNotNull("Should get some text but had none for file " + file, extractor.getText());
// also try metadata
@SuppressWarnings("resource")
Modified: poi/trunk/src/integrationtest/org/apache/poi/stress/FileHandler.java
URL: http://svn.apache.org/viewvc/poi/trunk/src/integrationtest/org/apache/poi/stress/FileHandler.java?rev=1787556&r1=1787555&r2=1787556&view=diff
==============================================================================
--- poi/trunk/src/integrationtest/org/apache/poi/stress/FileHandler.java (original)
+++ poi/trunk/src/integrationtest/org/apache/poi/stress/FileHandler.java Sat Mar 18 13:56:11 2017
@@ -31,8 +31,8 @@ public interface FileHandler {
*
* Closing is handled by the framework outside this call.
*
- * @param stream
- * @throws Exception
+ * @param stream The input stream to read the file from.
+ * @throws Exception If an error happens in the file-specific handler
*/
void handleFile(InputStream stream) throws Exception;
Modified: poi/trunk/src/integrationtest/org/apache/poi/stress/HSLFFileHandler.java
URL: http://svn.apache.org/viewvc/poi/trunk/src/integrationtest/org/apache/poi/stress/HSLFFileHandler.java?rev=1787556&r1=1787555&r2=1787556&view=diff
==============================================================================
--- poi/trunk/src/integrationtest/org/apache/poi/stress/HSLFFileHandler.java (original)
+++ poi/trunk/src/integrationtest/org/apache/poi/stress/HSLFFileHandler.java Sat Mar 18 13:56:11 2017
@@ -19,9 +19,7 @@ package org.apache.poi.stress;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStream;
+import java.io.*;
import org.apache.poi.hslf.record.Record;
import org.apache.poi.hslf.usermodel.HSLFSlideShow;
@@ -39,30 +37,65 @@ public class HSLFFileHandler extends Sli
Record[] records = slide.getRecords();
assertNotNull(records);
for(Record record : records) {
+ assertNotNull("Found a record which was null", record);
assertTrue(record.getRecordType() >= 0);
}
-
+
handlePOIDocument(slide);
-
+
HSLFSlideShow ss = new HSLFSlideShow(slide);
handleSlideShow(ss);
}
-
+
+ @Test
+ public void testOne() throws Exception {
+ testOneFile(new File("test-data/slideshow/54880_chinese.ppt"));
+ }
+
// a test-case to test this locally without executing the full TestAllFiles
@Override
@Test
public void test() throws Exception {
- InputStream stream = new FileInputStream("test-data/hpsf/Test_Humor-Generation.ppt");
- try {
- handleFile(stream);
- } finally {
- stream.close();
+ File[] files = new File("test-data/slideshow/").listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.endsWith(".ppt");
+ }
+ });
+ assertNotNull(files);
+
+ System.out.println("Testing " + files.length + " files");
+
+ for(File file : files) {
+ try {
+ testOneFile(file);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ }
}
+ }
+
+ private void testOneFile(File file) throws Exception {
+ System.out.println(file);
+
+ //System.setProperty("org.apache.poi.util.POILogger", "org.apache.poi.util.SystemOutLogger");
+ InputStream stream = new FileInputStream(file);
+ try {
+ handleFile(stream);
+ } finally {
+ stream.close();
+ }
+
+ handleExtracting(file);
}
- // a test-case to test this locally without executing the full TestAllFiles
- @Test
- public void testExtractor() throws Exception {
- handleExtracting(new File("test-data/slideshow/ae.ac.uaeu.faculty_nafaachbili_GeomLec1.pptx"));
+ public static void main(String[] args) throws Exception {
+ System.setProperty("org.apache.poi.util.POILogger", "org.apache.poi.util.SystemOutLogger");
+ InputStream stream = new FileInputStream(args[0]);
+ try {
+ new HSLFFileHandler().handleFile(stream);
+ } finally {
+ stream.close();
+ }
}
}
Modified: poi/trunk/src/integrationtest/org/apache/poi/stress/HSMFFileHandler.java
URL: http://svn.apache.org/viewvc/poi/trunk/src/integrationtest/org/apache/poi/stress/HSMFFileHandler.java?rev=1787556&r1=1787555&r2=1787556&view=diff
==============================================================================
--- poi/trunk/src/integrationtest/org/apache/poi/stress/HSMFFileHandler.java (original)
+++ poi/trunk/src/integrationtest/org/apache/poi/stress/HSMFFileHandler.java Sat Mar 18 13:56:11 2017
@@ -18,6 +18,7 @@ package org.apache.poi.stress;
import static org.junit.Assert.assertNotNull;
+import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
@@ -75,11 +76,14 @@ public class HSMFFileHandler extends POI
@Override
@Test
public void test() throws Exception {
- InputStream stream = new FileInputStream("test-data/hsmf/example_received_regular.msg");
+ File file = new File("test-data/hsmf/logsat.com_signatures_valid.msg");
+ InputStream stream = new FileInputStream(file);
try {
handleFile(stream);
} finally {
stream.close();
}
+
+ handleExtracting(file);
}
}
\ No newline at end of file
Modified: poi/trunk/src/integrationtest/org/apache/poi/stress/HSSFFileHandler.java
URL: http://svn.apache.org/viewvc/poi/trunk/src/integrationtest/org/apache/poi/stress/HSSFFileHandler.java?rev=1787556&r1=1787555&r2=1787556&view=diff
==============================================================================
--- poi/trunk/src/integrationtest/org/apache/poi/stress/HSSFFileHandler.java (original)
+++ poi/trunk/src/integrationtest/org/apache/poi/stress/HSSFFileHandler.java Sat Mar 18 13:56:11 2017
@@ -38,7 +38,7 @@ public class HSSFFileHandler extends Spr
// TODO: some documents fail currently...
// Note - as of Bugzilla 48036 (svn r828244, r828247) POI is capable of evaluating
- // IntesectionPtg. However it is still not capable of parsing it.
+ // IntersectionPtg. However it is still not capable of parsing it.
// So FormulaEvalTestData.xls now contains a few formulas that produce errors here.
//HSSFFormulaEvaluator evaluator = new HSSFFormulaEvaluator(wb);
//evaluator.evaluateAll();
@@ -100,17 +100,14 @@ public class HSSFFileHandler extends Spr
// a test-case to test this locally without executing the full TestAllFiles
@Test
public void test() throws Exception {
- InputStream stream = new FileInputStream("test-data/spreadsheet/49219.xls");
+ File file = new File("test-data/spreadsheet/49219.xls");
+
+ InputStream stream = new FileInputStream(file);
try {
handleFile(stream);
} finally {
stream.close();
}
+ handleExtracting(file);
}
-
- // a test-case to test this locally without executing the full TestAllFiles
- @Test
- public void testExtractor() throws Exception {
- handleExtracting(new File("test-data/spreadsheet/BOOK_in_capitals.xls"));
- }
}
\ No newline at end of file
Modified: poi/trunk/src/integrationtest/org/apache/poi/stress/OPCFileHandler.java
URL: http://svn.apache.org/viewvc/poi/trunk/src/integrationtest/org/apache/poi/stress/OPCFileHandler.java?rev=1787556&r1=1787555&r2=1787556&view=diff
==============================================================================
--- poi/trunk/src/integrationtest/org/apache/poi/stress/OPCFileHandler.java (original)
+++ poi/trunk/src/integrationtest/org/apache/poi/stress/OPCFileHandler.java Sat Mar 18 13:56:11 2017
@@ -57,7 +57,7 @@ public class OPCFileHandler extends Abst
@Override
public void handleExtracting(File file) throws Exception {
- // text-extraction is not possible currenlty for these types of files
+ // text-extraction is not possible currently for these types of files
}
// a test-case to test this locally without executing the full TestAllFiles
Modified: poi/trunk/src/integrationtest/org/apache/poi/stress/SlideShowHandler.java
URL: http://svn.apache.org/viewvc/poi/trunk/src/integrationtest/org/apache/poi/stress/SlideShowHandler.java?rev=1787556&r1=1787555&r2=1787556&view=diff
==============================================================================
--- poi/trunk/src/integrationtest/org/apache/poi/stress/SlideShowHandler.java (original)
+++ poi/trunk/src/integrationtest/org/apache/poi/stress/SlideShowHandler.java Sat Mar 18 13:56:11 2017
@@ -50,7 +50,7 @@ public abstract class SlideShowHandler e
readContent(ss);
- // read in the writen file
+ // read in the written file
SlideShow<?,?> read = SlideShowFactory.create(new ByteArrayInputStream(out.toByteArray()));
try {
assertNotNull(read);
Modified: poi/trunk/src/integrationtest/org/apache/poi/stress/XSLFFileHandler.java
URL: http://svn.apache.org/viewvc/poi/trunk/src/integrationtest/org/apache/poi/stress/XSLFFileHandler.java?rev=1787556&r1=1787555&r2=1787556&view=diff
==============================================================================
--- poi/trunk/src/integrationtest/org/apache/poi/stress/XSLFFileHandler.java (original)
+++ poi/trunk/src/integrationtest/org/apache/poi/stress/XSLFFileHandler.java Sat Mar 18 13:56:11 2017
@@ -69,17 +69,14 @@ public class XSLFFileHandler extends Sli
@Override
@Test
public void test() throws Exception {
- InputStream stream = new FileInputStream("test-data/slideshow/ae.ac.uaeu.faculty_nafaachbili_GeomLec1.pptx");
+ File file = new File("test-data/slideshow/ae.ac.uaeu.faculty_nafaachbili_GeomLec1.pptx");
+ InputStream stream = new FileInputStream(file);
try {
handleFile(stream);
} finally {
stream.close();
}
- }
- // a test-case to test this locally without executing the full TestAllFiles
- @Test
- public void testExtractor() throws Exception {
- handleExtracting(new File("test-data/slideshow/ae.ac.uaeu.faculty_nafaachbili_GeomLec1.pptx"));
+ handleExtracting(file);
}
-}
\ No newline at end of file
+}
Modified: poi/trunk/src/integrationtest/org/apache/poi/stress/XSSFFileHandler.java
URL: http://svn.apache.org/viewvc/poi/trunk/src/integrationtest/org/apache/poi/stress/XSSFFileHandler.java?rev=1787556&r1=1787555&r2=1787556&view=diff
==============================================================================
--- poi/trunk/src/integrationtest/org/apache/poi/stress/XSSFFileHandler.java (original)
+++ poi/trunk/src/integrationtest/org/apache/poi/stress/XSSFFileHandler.java Sat Mar 18 13:56:11 2017
@@ -16,25 +16,6 @@
==================================================================== */
package org.apache.poi.stress;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.TransformerException;
-import java.io.BufferedInputStream;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PrintStream;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Locale;
-import java.util.Set;
-
import org.apache.poi.POIXMLException;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.openxml4j.exceptions.OLE2NotOfficeXmlFileException;
@@ -56,12 +37,18 @@ public class XSSFFileHandler extends Spr
// ignore password protected files
if (POIXMLDocumentHandler.isEncrypted(stream)) return;
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- IOUtils.copy(stream, out);
-
- final byte[] bytes = out.toByteArray();
final XSSFWorkbook wb;
- wb = new XSSFWorkbook(new ByteArrayInputStream(bytes));
+
+ // make sure the potentially large byte-array is freed up quickly again
+ {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ IOUtils.copy(stream, out);
+ final byte[] bytes = out.toByteArray();
+
+ checkXSSFReader(OPCPackage.open(new ByteArrayInputStream(bytes)));
+
+ wb = new XSSFWorkbook(new ByteArrayInputStream(bytes));
+ }
// use the combined handler for HSSF/XSSF
handleWorkbook(wb);
@@ -76,9 +63,8 @@ public class XSSFFileHandler extends Spr
// and finally ensure that exporting to XML works
exportToXML(wb);
- checkXSSFReader(OPCPackage.open(new ByteArrayInputStream(bytes)));
-
- wb.close();
+ // this allows to trigger a heap-dump at this point to see which memory is still allocated
+ //HeapDump.dumpHeap("/tmp/poi.hprof", false);
}
@@ -183,18 +169,16 @@ public class XSSFFileHandler extends Spr
// a test-case to test this locally without executing the full TestAllFiles
@Test
public void test() throws Exception {
- InputStream stream = new BufferedInputStream(new FileInputStream("test-data/spreadsheet/ref-56737.xlsx"));
+ File file = new File("test-data/spreadsheet/ref-56737.xlsx");
+
+ InputStream stream = new BufferedInputStream(new FileInputStream(file));
try {
handleFile(stream);
} finally {
stream.close();
}
- }
- // a test-case to test this locally without executing the full TestAllFiles
- @Test
- public void testExtractor() throws Exception {
- handleExtracting(new File("test-data/spreadsheet/ref-56737.xlsx"));
+ handleExtracting(file);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@poi.apache.org
For additional commands, e-mail: commits-help@poi.apache.org