You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2012/12/28 07:31:08 UTC
svn commit: r1426414 - in /accumulo/trunk/core/src:
main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
main/java/org/apache/accumulo/core/file/rfile/RFile.java
test/java/org/apache/accumulo/core/file/rfile/BlockIndexTest.java
Author: kturner
Date: Fri Dec 28 06:31:08 2012
New Revision: 1426414
URL: http://svn.apache.org/viewvc?rev=1426414&view=rev
Log:
ACCUMULO-863 reduced block index entry to just one key
Added:
accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/BlockIndexTest.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java?rev=1426414&r1=1426413&r2=1426414&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/BlockIndex.java Fri Dec 28 06:31:08 2012
@@ -57,15 +57,13 @@ public class BlockIndex {
public static class BlockIndexEntry implements Comparable<BlockIndexEntry> {
- private Key key;
private Key prevKey;
private int entriesLeft;
private int pos;
- public BlockIndexEntry(int pos, int entriesLeft, Key key, Key prevKey) {
+ public BlockIndexEntry(int pos, int entriesLeft, Key prevKey) {
this.pos = pos;
this.entriesLeft = entriesLeft;
- this.key = key;
this.prevKey = prevKey;
}
@@ -73,12 +71,10 @@ public class BlockIndex {
* @param key
*/
public BlockIndexEntry(Key key) {
- this.key = key;
+ this.prevKey = key;
}
- public Key getKey() {
- return key;
- }
+
public int getEntriesLeft() {
return entriesLeft;
@@ -86,12 +82,12 @@ public class BlockIndex {
@Override
public int compareTo(BlockIndexEntry o) {
- return key.compareTo(o.key);
+ return prevKey.compareTo(o.prevKey);
}
@Override
public String toString() {
- return key + " " + entriesLeft + " " + pos;
+ return prevKey + " " + entriesLeft + " " + pos;
}
public Key getPrevKey() {
@@ -116,18 +112,24 @@ public class BlockIndex {
} else {
// found exact key in index
index = pos;
+ while (index > 0) {
+ if (blockIndex[index].getPrevKey().equals(startKey))
+ index--;
+ else
+ break;
+ }
}
// handle case where multiple keys in block are exactly the same, want to find the earliest key in the index
while (index - 1 > 0) {
- if (blockIndex[index].getKey().equals(blockIndex[index - 1].getKey()))
+ if (blockIndex[index].getPrevKey().equals(blockIndex[index - 1].getPrevKey()))
index--;
else
break;
}
- if (index == 0 && blockIndex[index].getKey().equals(startKey))
+ if (index == 0 && blockIndex[index].getPrevKey().equals(startKey))
return null;
BlockIndexEntry bie = blockIndex[index];
@@ -162,7 +164,7 @@ public class BlockIndex {
val.readFields(cacheBlock);
if (count > 0 && count % interval == 0) {
- index.add(new BlockIndexEntry(pos, indexEntry.getNumEntries() - count, rk.getKey(), myPrevKey));
+ index.add(new BlockIndexEntry(pos, indexEntry.getNumEntries() - count, myPrevKey));
}
count++;
@@ -172,4 +174,8 @@ public class BlockIndex {
cacheBlock.seek(0);
}
+
+ BlockIndexEntry[] getIndexEntries() {
+ return blockIndex;
+ }
}
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java?rev=1426414&r1=1426413&r2=1426414&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java Fri Dec 28 06:31:08 2012
@@ -751,7 +751,7 @@ public class RFile {
// just consumed one key from the input stream, so subtract one from entries left
entriesLeft = bie.getEntriesLeft() - 1;
prevKey = new Key(bie.getPrevKey());
- currKey = bie.getKey();
+ currKey = tmpRk.getKey();
}
}
}
Added: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/BlockIndexTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/BlockIndexTest.java?rev=1426414&view=auto
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/BlockIndexTest.java (added)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/rfile/BlockIndexTest.java Fri Dec 28 06:31:08 2012
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.rfile;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.blockfile.ABlockReader;
+import org.apache.accumulo.core.file.blockfile.cache.CacheEntry;
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+import org.apache.accumulo.core.file.rfile.BlockIndex.BlockIndexEntry;
+import org.apache.accumulo.core.file.rfile.MultiLevelIndex.IndexEntry;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class BlockIndexTest {
+
+ private static class MyCacheEntry implements CacheEntry {
+ Object idx;
+ byte[] data;
+
+ MyCacheEntry(byte[] d) {
+ this.data = d;
+ }
+
+ @Override
+ public void setIndex(Object idx) {
+ this.idx = idx;
+ }
+
+ @Override
+ public Object getIndex() {
+ return idx;
+ }
+
+ @Override
+ public byte[] getBuffer() {
+ return data;
+ }
+ }
+
+ @Test
+ public void test1() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
+
+ Key prevKey = null;
+
+ int num = 1000;
+
+ for (int i = 0; i < num; i++) {
+ Key key = new Key(RFileTest.nf("", i), "cf1", "cq1");
+ new RelativeKey(prevKey, key).write(out);
+ new Value(new byte[0]).write(out);
+ prevKey = key;
+ }
+
+ out.close();
+ final byte[] data = baos.toByteArray();
+
+ CacheEntry ce = new MyCacheEntry(data);
+
+ ABlockReader cacheBlock = new CachableBlockFile.CachedBlockRead(ce, data);
+ BlockIndex blockIndex = null;
+
+ for (int i = 0; i < 129; i++)
+ blockIndex = BlockIndex.getIndex(cacheBlock, new IndexEntry(prevKey, num, 0, 0, 0));
+
+ BlockIndexEntry[] indexEntries = blockIndex.getIndexEntries();
+
+ for (int i = 0; i < indexEntries.length; i++) {
+ int row = Integer.parseInt(indexEntries[i].getPrevKey().getRowData().toString());
+
+ BlockIndexEntry bie;
+
+
+ bie = blockIndex.seekBlock(new Key(RFileTest.nf("", row), "cf1", "cq1"), cacheBlock);
+ if (i == 0)
+ Assert.assertSame(null, bie);
+ else
+ Assert.assertSame(indexEntries[i - 1], bie);
+
+ Assert.assertSame(bie, blockIndex.seekBlock(new Key(RFileTest.nf("", row - 1), "cf1", "cq1"), cacheBlock));
+
+ bie = blockIndex.seekBlock(new Key(RFileTest.nf("", row + 1), "cf1", "cq1"), cacheBlock);
+ Assert.assertSame(indexEntries[i], bie);
+
+ RelativeKey rk = new RelativeKey();
+ rk.setPrevKey(bie.getPrevKey());
+ rk.readFields(cacheBlock);
+
+ Assert.assertEquals(rk.getKey(), new Key(RFileTest.nf("", row + 1), "cf1", "cq1"));
+
+ }
+ }
+
+ @Test
+ public void testSame() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
+
+ Key prevKey = null;
+
+ int num = 1000;
+
+ for (int i = 0; i < num; i++) {
+ Key key = new Key(RFileTest.nf("", 1), "cf1", "cq1");
+ new RelativeKey(prevKey, key).write(out);
+ new Value(new byte[0]).write(out);
+ prevKey = key;
+ }
+
+ for (int i = 0; i < num; i++) {
+ Key key = new Key(RFileTest.nf("", 3), "cf1", "cq1");
+ new RelativeKey(prevKey, key).write(out);
+ new Value(new byte[0]).write(out);
+ prevKey = key;
+ }
+
+ for (int i = 0; i < num; i++) {
+ Key key = new Key(RFileTest.nf("", 5), "cf1", "cq1");
+ new RelativeKey(prevKey, key).write(out);
+ new Value(new byte[0]).write(out);
+ prevKey = key;
+ }
+
+ out.close();
+ final byte[] data = baos.toByteArray();
+
+ CacheEntry ce = new MyCacheEntry(data);
+
+ ABlockReader cacheBlock = new CachableBlockFile.CachedBlockRead(ce, data);
+ BlockIndex blockIndex = null;
+
+ for (int i = 0; i < 257; i++)
+ blockIndex = BlockIndex.getIndex(cacheBlock, new IndexEntry(prevKey, num, 0, 0, 0));
+
+ Assert.assertSame(null, blockIndex.seekBlock(new Key(RFileTest.nf("", 0), "cf1", "cq1"), cacheBlock));
+ Assert.assertSame(null, blockIndex.seekBlock(new Key(RFileTest.nf("", 1), "cf1", "cq1"), cacheBlock));
+
+ for (int i = 2; i < 6; i++) {
+ Key seekKey = new Key(RFileTest.nf("", i), "cf1", "cq1");
+ BlockIndexEntry bie = blockIndex.seekBlock(seekKey, cacheBlock);
+
+ Assert.assertTrue(bie.getPrevKey().compareTo(seekKey) < 0);
+
+ RelativeKey rk = new RelativeKey();
+ rk.setPrevKey(bie.getPrevKey());
+ rk.readFields(cacheBlock);
+
+ Assert.assertTrue(rk.getKey().compareTo(seekKey) <= 0);
+ }
+
+ }
+}