You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@poi.apache.org by fa...@apache.org on 2021/10/04 20:34:47 UTC
svn commit: r1893896 - in /poi/trunk/poi-ooxml/src:
main/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java
test/java/org/apache/poi/xssf/DeferredSXSSFITestDataProvider.java
test/java/org/apache/poi/xssf/streaming/TestDeferredSXSSFWorkbook.java
Author: fanningpj
Date: Mon Oct 4 20:34:47 2021
New Revision: 1893896
URL: http://svn.apache.org/viewvc?rev=1893896&view=rev
Log:
[bug-65613] experimental SXSSF writeAvoidingTempFiles
Modified:
poi/trunk/poi-ooxml/src/main/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java
poi/trunk/poi-ooxml/src/test/java/org/apache/poi/xssf/DeferredSXSSFITestDataProvider.java
poi/trunk/poi-ooxml/src/test/java/org/apache/poi/xssf/streaming/TestDeferredSXSSFWorkbook.java
Modified: poi/trunk/poi-ooxml/src/main/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java
URL: http://svn.apache.org/viewvc/poi/trunk/poi-ooxml/src/main/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java?rev=1893896&r1=1893895&r2=1893896&view=diff
==============================================================================
--- poi/trunk/poi-ooxml/src/main/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java (original)
+++ poi/trunk/poi-ooxml/src/main/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java Mon Oct 4 20:34:47 2021
@@ -17,13 +17,7 @@
package org.apache.poi.xssf.streaming;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
+import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.Enumeration;
import java.util.HashMap;
@@ -35,13 +29,16 @@ import java.util.NoSuchElementException;
import org.apache.commons.compress.archivers.ArchiveOutputStream;
import org.apache.commons.compress.archivers.zip.Zip64Mode;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.openxml4j.util.ZipArchiveThresholdInputStream;
import org.apache.poi.openxml4j.util.ZipEntrySource;
import org.apache.poi.openxml4j.util.ZipFileZipEntrySource;
+import org.apache.poi.openxml4j.util.ZipInputStreamZipEntrySource;
import org.apache.poi.openxml4j.util.ZipSecureFile;
import org.apache.poi.ss.SpreadsheetVersion;
import org.apache.poi.ss.formula.EvaluationWorkbook;
@@ -403,8 +400,8 @@ public class SXSSFWorkbook implements Wo
while (en.hasMoreElements()) {
ZipArchiveEntry ze = en.nextElement();
ZipArchiveEntry zeOut = new ZipArchiveEntry(ze.getName());
- zeOut.setSize(ze.getSize());
- zeOut.setTime(ze.getTime());
+ if (ze.getSize() >= 0) zeOut.setSize(ze.getSize());
+ if (ze.getTime() >= 0) zeOut.setTime(ze.getTime());
zos.putArchiveEntry(zeOut);
try (final InputStream is = zipEntrySource.getInputStream(ze)) {
if (is instanceof ZipArchiveThresholdInputStream) {
@@ -968,11 +965,38 @@ public class SXSSFWorkbook implements Wo
} finally {
deleted = tmplFile.delete();
}
- if(!deleted) {
+ if (!deleted) {
throw new IOException("Could not delete temporary file after processing: " + tmplFile);
}
}
+ /**
+ * Write out this workbook to an OutputStream. This (experimental) method avoids the temp file that
+ * {@link #write} creates but will use more memory as a result. Other SXSSF code can create temp files,
+ * so using this does not guarantee that there will be no temp file usage.
+ *
+ * @param stream - the java OutputStream you wish to write to
+ * @exception IOException if anything can't be written.
+ */
+ @Beta
+ public void writeAvoidingTempFiles(OutputStream stream) throws IOException {
+ flushSheets();
+
+ //Save the template
+ try (UnsynchronizedByteArrayOutputStream bos = new UnsynchronizedByteArrayOutputStream()) {
+ _wb.write(bos);
+
+ //Substitute the template entries with the generated sheet data files
+ try (
+ InputStream is = bos.toInputStream();
+ ZipInputStreamZipEntrySource source = new ZipInputStreamZipEntrySource(
+ new ZipArchiveThresholdInputStream(new ZipArchiveInputStream(is)))
+ ) {
+ injectData(source, stream);
+ }
+ }
+ }
+
protected void flushSheets() throws IOException {
for (SXSSFSheet sheet : _xFromSxHash.values())
{
Modified: poi/trunk/poi-ooxml/src/test/java/org/apache/poi/xssf/DeferredSXSSFITestDataProvider.java
URL: http://svn.apache.org/viewvc/poi/trunk/poi-ooxml/src/test/java/org/apache/poi/xssf/DeferredSXSSFITestDataProvider.java?rev=1893896&r1=1893895&r2=1893896&view=diff
==============================================================================
--- poi/trunk/poi-ooxml/src/test/java/org/apache/poi/xssf/DeferredSXSSFITestDataProvider.java (original)
+++ poi/trunk/poi-ooxml/src/test/java/org/apache/poi/xssf/DeferredSXSSFITestDataProvider.java Mon Oct 4 20:34:47 2021
@@ -78,6 +78,20 @@ public final class DeferredSXSSFITestDat
}
}
+ /**
+ * Returns an XSSFWorkbook since SXSSFWorkbook is write-only
+ */
+ public XSSFWorkbook inMemoryWriteOutAndReadBack(SXSSFWorkbook wb) {
+ try (UnsynchronizedByteArrayOutputStream baos = new UnsynchronizedByteArrayOutputStream()) {
+ wb.writeAvoidingTempFiles(baos);
+ try (InputStream is = baos.toInputStream()) {
+ return new XSSFWorkbook(is);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
@Override
public DeferredSXSSFWorkbook createWorkbook() {
DeferredSXSSFWorkbook wb = new DeferredSXSSFWorkbook();
Modified: poi/trunk/poi-ooxml/src/test/java/org/apache/poi/xssf/streaming/TestDeferredSXSSFWorkbook.java
URL: http://svn.apache.org/viewvc/poi/trunk/poi-ooxml/src/test/java/org/apache/poi/xssf/streaming/TestDeferredSXSSFWorkbook.java?rev=1893896&r1=1893895&r2=1893896&view=diff
==============================================================================
--- poi/trunk/poi-ooxml/src/test/java/org/apache/poi/xssf/streaming/TestDeferredSXSSFWorkbook.java (original)
+++ poi/trunk/poi-ooxml/src/test/java/org/apache/poi/xssf/streaming/TestDeferredSXSSFWorkbook.java Mon Oct 4 20:34:47 2021
@@ -188,6 +188,84 @@ public final class TestDeferredSXSSFWork
}
@Test
+ void inMemoryWrite() throws IOException {
+ try (XSSFWorkbook xssfWb1 = new XSSFWorkbook()) {
+ xssfWb1.createSheet("S1");
+ Sheet sheet = xssfWb1.createSheet("S2");
+ Row row = sheet.createRow(1);
+ Cell cell = row.createCell(1);
+ cell.setCellValue("value 2_1_1");
+
+ try (DeferredSXSSFWorkbook wb1 = new DeferredSXSSFWorkbook(xssfWb1);
+ XSSFWorkbook xssfWb2 = DeferredSXSSFITestDataProvider.instance.writeOutAndReadBack(wb1)) {
+ assertTrue(wb1.dispose());
+
+ try (DeferredSXSSFWorkbook wb2 = new DeferredSXSSFWorkbook(xssfWb2)) {
+ // Add a row to the existing empty sheet
+ DeferredSXSSFSheet ssheet1 = wb2.getStreamingSheetAt(0);
+ ssheet1.setRowGenerator((ssxSheet) -> {
+ Row row1_1 = ssxSheet.createRow(1);
+ Cell cell1_1_1 = row1_1.createCell(1);
+ cell1_1_1.setCellValue("value 1_1_1");
+ });
+
+ // Add a row to the existing non-empty sheet
+ DeferredSXSSFSheet ssheet2 = wb2.getStreamingSheetAt(1);
+ ssheet2.setRowGenerator((ssxSheet) -> {
+ Row row2_2 = ssxSheet.createRow(2);
+ Cell cell2_2_1 = row2_2.createCell(1);
+ cell2_2_1.setCellValue("value 2_2_1");
+ });
+ // Add a sheet with one row
+ DeferredSXSSFSheet ssheet3 = wb2.createSheet("S3");
+ ssheet3.setRowGenerator((ssxSheet) -> {
+ Row row3_1 = ssxSheet.createRow(1);
+ Cell cell3_1_1 = row3_1.createCell(1);
+ cell3_1_1.setCellValue("value 3_1_1");
+ });
+
+ try (XSSFWorkbook xssfWb3 = DeferredSXSSFITestDataProvider.instance.inMemoryWriteOutAndReadBack(wb2)) {
+
+ assertEquals(3, xssfWb3.getNumberOfSheets());
+ // Verify sheet 1
+ XSSFSheet sheet1 = xssfWb3.getSheetAt(0);
+ assertEquals("S1", sheet1.getSheetName());
+ assertEquals(1, sheet1.getPhysicalNumberOfRows());
+ XSSFRow row1_1 = sheet1.getRow(1);
+ assertNotNull(row1_1);
+ XSSFCell cell1_1_1 = row1_1.getCell(1);
+ assertNotNull(cell1_1_1);
+ assertEquals("value 1_1_1", cell1_1_1.getStringCellValue());
+ // Verify sheet 2
+ XSSFSheet sheet2 = xssfWb3.getSheetAt(1);
+ assertEquals("S2", sheet2.getSheetName());
+ assertEquals(2, sheet2.getPhysicalNumberOfRows());
+ Row row2_1 = sheet2.getRow(1);
+ assertNotNull(row2_1);
+ Cell cell2_1_1 = row2_1.getCell(1);
+ assertNotNull(cell2_1_1);
+ assertEquals("value 2_1_1", cell2_1_1.getStringCellValue());
+ XSSFRow row2_2 = sheet2.getRow(2);
+ assertNotNull(row2_2);
+ XSSFCell cell2_2_1 = row2_2.getCell(1);
+ assertNotNull(cell2_2_1);
+ assertEquals("value 2_2_1", cell2_2_1.getStringCellValue());
+ // Verify sheet 3
+ XSSFSheet sheet3 = xssfWb3.getSheetAt(2);
+ assertEquals("S3", sheet3.getSheetName());
+ assertEquals(1, sheet3.getPhysicalNumberOfRows());
+ XSSFRow row3_1 = sheet3.getRow(1);
+ assertNotNull(row3_1);
+ XSSFCell cell3_1_1 = row3_1.getCell(1);
+ assertNotNull(cell3_1_1);
+ assertEquals("value 3_1_1", cell3_1_1.getStringCellValue());
+ }
+ }
+ }
+ }
+ }
+
+ @Test
void sheetdataWriter() throws IOException {
try (DeferredSXSSFWorkbook wb = new DeferredSXSSFWorkbook()) {
SXSSFSheet sh = wb.createSheet();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@poi.apache.org
For additional commands, e-mail: commits-help@poi.apache.org