You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uniffle.apache.org by js...@apache.org on 2022/07/01 06:57:59 UTC
[incubator-uniffle] branch branch-0.3.0 created (now 1d69058)
This is an automated email from the ASF dual-hosted git repository.
jshao pushed a change to branch branch-0.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
at 1d69058 [Bugfix] Fix uncorrect index file (#92) (#93)
This branch includes the following new commits:
new 1d69058 [Bugfix] Fix uncorrect index file (#92) (#93)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-uniffle] 01/01: [Bugfix] Fix uncorrect index file (#92) (#93)
Posted by js...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch branch-0.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
commit 1d69058c32f8f943e1694cfe182fb19d55943a11
Author: roryqi <je...@gmail.com>
AuthorDate: Tue Mar 8 17:21:55 2022 +0800
[Bugfix] Fix uncorrect index file (#92) (#93)
backport 0.3.0
### What changes were proposed in this pull request?
Modify the method that calculate the offset in the index file.
### Why are the changes needed?
If we don't have this patch, we run 10TB tpcds, query24a will fail.
<img width="361" alt="企业微信截图_6dc451cf-dbf4-4257-b680-e79346cd582d" src="https://user-images.githubusercontent.com/8159038/157178756-d8a39b3f-0ea6-4864-ac68-ee382a88bb0f.png">
When we write many data to dataOutputStream, dataOutputStream.size() won't increase again. dataOutputStream.size() will
always be Integer.MAX_VALUE.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Add new uts.
Co-authored-by: roryqi <ro...@tencent.com>
---
.../rss/storage/handler/impl/LocalFileWriter.java | 6 ++----
.../rss/storage/handler/impl/LocalFileHandlerTest.java | 17 +++++++++++++++++
2 files changed, 19 insertions(+), 4 deletions(-)
diff --git a/storage/src/main/java/com/tencent/rss/storage/handler/impl/LocalFileWriter.java b/storage/src/main/java/com/tencent/rss/storage/handler/impl/LocalFileWriter.java
index 10185a4..609db7e 100644
--- a/storage/src/main/java/com/tencent/rss/storage/handler/impl/LocalFileWriter.java
+++ b/storage/src/main/java/com/tencent/rss/storage/handler/impl/LocalFileWriter.java
@@ -30,21 +30,19 @@ public class LocalFileWriter implements Closeable {
private DataOutputStream dataOutputStream;
private FileOutputStream fileOutputStream;
- private long initSize;
private long nextOffset;
public LocalFileWriter(File file) throws IOException {
fileOutputStream = new FileOutputStream(file, true);
// init fsDataOutputStream
dataOutputStream = new DataOutputStream(fileOutputStream);
- initSize = file.length();
- nextOffset = initSize;
+ nextOffset = file.length();
}
public void writeData(byte[] data) throws IOException {
if (data != null && data.length > 0) {
dataOutputStream.write(data);
- nextOffset = initSize + dataOutputStream.size();
+ nextOffset = nextOffset + data.length;
}
}
diff --git a/storage/src/test/java/com/tencent/rss/storage/handler/impl/LocalFileHandlerTest.java b/storage/src/test/java/com/tencent/rss/storage/handler/impl/LocalFileHandlerTest.java
index 32b7ace..846ab20 100644
--- a/storage/src/test/java/com/tencent/rss/storage/handler/impl/LocalFileHandlerTest.java
+++ b/storage/src/test/java/com/tencent/rss/storage/handler/impl/LocalFileHandlerTest.java
@@ -39,6 +39,7 @@ import com.tencent.rss.storage.handler.api.ServerReadHandler;
import com.tencent.rss.storage.handler.api.ShuffleWriteHandler;
import com.tencent.rss.storage.util.ShuffleStorageUtils;
import java.io.File;
+import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Random;
@@ -53,6 +54,7 @@ public class LocalFileHandlerTest {
@Test
public void writeTest() throws Exception {
File tmpDir = Files.createTempDir();
+ tmpDir.deleteOnExit();
File dataDir1 = new File(tmpDir, "data1");
File dataDir2 = new File(tmpDir, "data2");
String[] basePaths = new String[]{dataDir1.getAbsolutePath(),
@@ -111,6 +113,21 @@ public class LocalFileHandlerTest {
}
}
+ @Test
+ public void writeBigDataTest() throws IOException {
+ File tmpDir = Files.createTempDir();
+ tmpDir.deleteOnExit();
+ File writeFile = new File(tmpDir, "writetest");
+ LocalFileWriter writer = new LocalFileWriter(writeFile);
+ int size = Integer.MAX_VALUE / 100;
+ byte[] data = new byte[size];
+ for (int i = 0; i < 200; i++) {
+ writer.writeData(data);
+ }
+ long totalSize = 200L * size;
+ assertEquals(writer.nextOffset(), totalSize);
+ }
+
private void writeTestData(
ShuffleWriteHandler writeHandler,