[GitHub] merlimat closed pull request #1738: PIP-17: fix index entry offset error in OffloadIndexBlock

[GitBox <gi...@apache.org>]

merlimat closed pull request #1738: PIP-17:  fix index entry offset error in OffloadIndexBlock
URL: https://github.com/apache/incubator-pulsar/pull/1738
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockBuilderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockBuilderImpl.java
index ced3bf37c6..766083dcb3 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockBuilderImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockBuilderImpl.java
@@ -35,6 +35,7 @@
 
     private LedgerMetadata ledgerMetadata;
     private List<OffloadIndexEntryImpl> entries;
+    private int lastBlockSize;
 
     public OffloadIndexBlockBuilderImpl() {
         this.entries = Lists.newArrayList();
@@ -50,13 +51,14 @@ public OffloadIndexBlockBuilder withMetadata(LedgerMetadata metadata) {
     public OffloadIndexBlockBuilder addBlock(long firstEntryId, int partId, int blockSize) {
         // we should added one by one.
         long offset;
-        if(firstEntryId == 0) {
+        if (firstEntryId == 0) {
             checkState(entries.size() == 0);
             offset = 0;
         } else {
             checkState(entries.size() > 0);
-            offset = entries.get(entries.size() - 1).getOffset() + blockSize;
+            offset = entries.get(entries.size() - 1).getOffset() + lastBlockSize;
         }
+        lastBlockSize = blockSize;
 
         this.entries.add(OffloadIndexEntryImpl.of(firstEntryId, partId, offset));
         return this;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockImpl.java
index 31058b4f29..086d466879 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockImpl.java
@@ -96,7 +96,7 @@ public void recycle() {
 
     @Override
     public OffloadIndexEntry getIndexEntryForEntry(long messageEntryId) throws IOException {
-        if(messageEntryId > segmentMetadata.getLastEntryId()) {
+        if (messageEntryId > segmentMetadata.getLastEntryId()) {
             log.warn("Try to get entry: {}, which beyond lastEntryId {}, return null",
                 messageEntryId, segmentMetadata.getLastEntryId());
             throw new IndexOutOfBoundsException("Entry index: " + messageEntryId +
@@ -302,7 +302,8 @@ private OffloadIndexBlock fromStream(InputStream stream) throws IOException {
         DataInputStream dis = new DataInputStream(stream);
         int magic = dis.readInt();
         if (magic != this.INDEX_MAGIC_WORD) {
-            throw new IOException("Invalid MagicWord. read: " + magic + " expected: " + INDEX_MAGIC_WORD);
+            throw new IOException("Invalid MagicWord. read: " + Integer.toHexString(magic)
+                + " expected: 0x" + Integer.toHexString(INDEX_MAGIC_WORD));
         }
         int indexBlockLength = dis.readInt();
         int segmentMetadataLength = dis.readInt();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexTest.java
index c0ae5ed993..3867899b57 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexTest.java
@@ -110,7 +110,7 @@ public void offloadIndexBlockImplTest() throws Exception {
 
         blockBuilder.withMetadata(metadata);
 
-        blockBuilder.addBlock(0, 2, 0);
+        blockBuilder.addBlock(0, 2, 64 * 1024 * 1024);
         blockBuilder.addBlock(1000, 3, 64 * 1024 * 1024);
         blockBuilder.addBlock(2000, 4, 64 * 1024 * 1024);
         OffloadIndexBlock indexBlock = blockBuilder.build();


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services