You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2016/03/02 08:25:15 UTC
hbase git commit: HBASE-15359 Simplifying segment hierarchy
Repository: hbase
Updated Branches:
refs/heads/master 3fa77a184 -> dc4494366
HBASE-15359 Simplifying segment hierarchy
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dc449436
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dc449436
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dc449436
Branch: refs/heads/master
Commit: dc449436662d8520a02628c438d0da665ab6da2e
Parents: 3fa77a1
Author: eshcar <es...@yahoo-inc.com>
Authored: Tue Mar 1 10:25:39 2016 +0200
Committer: stack <st...@apache.org>
Committed: Tue Mar 1 23:25:11 2016 -0800
----------------------------------------------------------------------
.../hbase/regionserver/AbstractMemStore.java | 34 +--
.../hbase/regionserver/DefaultMemStore.java | 14 -
.../hbase/regionserver/ImmutableSegment.java | 40 +--
.../regionserver/ImmutableSegmentAdapter.java | 107 --------
.../regionserver/MutableCellSetSegment.java | 153 -----------
.../MutableCellSetSegmentScanner.java | 258 -------------------
.../hbase/regionserver/MutableSegment.java | 39 ++-
.../hadoop/hbase/regionserver/Segment.java | 118 +++++++--
.../hbase/regionserver/SegmentFactory.java | 9 +-
.../hbase/regionserver/SegmentScanner.java | 230 ++++++++++++++++-
10 files changed, 371 insertions(+), 631 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc449436/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
index 18d2f8a..08e3f5e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
@@ -226,6 +226,23 @@ public abstract class AbstractMemStore implements MemStore {
return getSnapshot().getSize();
}
+ /**
+ * Remove n key from the memstore. Only cells that have the same key and the
+ * same memstoreTS are removed. It is ok to not update timeRangeTracker
+ * in this call. It is possible that we can optimize this method by using
+ * tailMap/iterator, but since this method is called rarely (only for
+ * error recovery), we can leave those optimization for the future.
+ * @param cell
+ */
+ @Override
+ public void rollback(Cell cell) {
+ // If the key is in the active, delete it. Update this.size.
+ long sz = active.rollback(cell);
+ if (sz != 0) {
+ setOldestEditTimeToNow();
+ }
+ }
+
@Override
public String toString() {
StringBuffer buf = new StringBuffer();
@@ -241,23 +258,6 @@ public abstract class AbstractMemStore implements MemStore {
return buf.toString();
}
- protected void rollbackInSnapshot(Cell cell) {
- // If the key is in the snapshot, delete it. We should not update
- // this.size, because that tracks the size of only the memstore and
- // not the snapshot. The flush of this snapshot to disk has not
- // yet started because Store.flush() waits for all rwcc transactions to
- // commit before starting the flush to disk.
- snapshot.rollback(cell);
- }
-
- protected void rollbackInActive(Cell cell) {
- // If the key is in the memstore, delete it. Update this.size.
- long sz = active.rollback(cell);
- if (sz != 0) {
- setOldestEditTimeToNow();
- }
- }
-
protected Configuration getConfiguration() {
return conf;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc449436/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
index 92bb7b6..1bb9343 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
@@ -118,20 +118,6 @@ public class DefaultMemStore extends AbstractMemStore {
}
/**
- * Remove n key from the memstore. Only cells that have the same key and the
- * same memstoreTS are removed. It is ok to not update timeRangeTracker
- * in this call. It is possible that we can optimize this method by using
- * tailMap/iterator, but since this method is called rarely (only for
- * error recovery), we can leave those optimization for the future.
- * @param cell
- */
- @Override
- public void rollback(Cell cell) {
- rollbackInSnapshot(cell);
- rollbackInActive(cell);
- }
-
- /**
* @param cell Find the row that comes after this one. If null, we return the
* first.
* @return Next row or null if none found.
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc449436/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java
index cfcd81e..077c270 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java
@@ -18,55 +18,29 @@
*/
package org.apache.hadoop.hbase.regionserver;
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.CollectionBackedScanner;
/**
* ImmutableSegment is an abstract class that extends the API supported by a {@link Segment},
* and is not needed for a {@link MutableSegment}. Specifically, the method
* {@link ImmutableSegment#getKeyValueScanner()} builds a special scanner for the
* {@link MemStoreSnapshot} object.
- * In addition, this class overrides methods that are not likely to be supported by an immutable
- * segment, e.g. {@link Segment#rollback(Cell)} and {@link Segment#getCellSet()}, which
- * can be very inefficient.
*/
@InterfaceAudience.Private
-public abstract class ImmutableSegment extends Segment {
+public class ImmutableSegment extends Segment {
- public ImmutableSegment(Segment segment) {
+ protected ImmutableSegment(Segment segment) {
super(segment);
}
/**
- * Removes the given cell from this segment.
- * By default immutable store segment can not rollback
- * It may be invoked by tests in specific cases where it is known to be supported {@link
- * ImmutableSegmentAdapter}
- */
- @Override
- public long rollback(Cell cell) {
- return 0;
- }
-
- /**
- * Returns a set of all the cells in the segment.
- * The implementation of this method might be very inefficient for some immutable segments
- * that do not maintain a cell set. Therefore by default this method is not supported.
- * It may be invoked by tests in specific cases where it is known to be supported {@link
- * ImmutableSegmentAdapter}
- */
- @Override
- public CellSet getCellSet() {
- throw new NotImplementedException("Immutable Segment does not support this operation by " +
- "default");
- }
-
- /**
- * Builds a special scanner for the MemStoreSnapshot object that may be different than the
+ * Builds a special scanner for the MemStoreSnapshot object that is different than the
* general segment scanner.
* @return a special scanner for the MemStoreSnapshot object
*/
- public abstract KeyValueScanner getKeyValueScanner();
+ public KeyValueScanner getKeyValueScanner() {
+ return new CollectionBackedScanner(getCellSet(), getComparator());
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc449436/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegmentAdapter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegmentAdapter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegmentAdapter.java
deleted file mode 100644
index 058865a..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegmentAdapter.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import org.apache.commons.logging.Log;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.util.CollectionBackedScanner;
-
-/**
- * This segment is adapting a mutable segment making it into an immutable segment.
- * This is used when a mutable segment is moved to being a snapshot or pushed into a compaction
- * pipeline, that consists only of immutable segments.
- * The compaction may generate different type of immutable segment
- */
-@InterfaceAudience.Private
-public class ImmutableSegmentAdapter extends ImmutableSegment {
-
- final private MutableSegment adaptee;
-
- public ImmutableSegmentAdapter(MutableSegment segment) {
- super(segment);
- this.adaptee = segment;
- }
-
- @Override
- public KeyValueScanner getKeyValueScanner() {
- return new CollectionBackedScanner(adaptee.getCellSet(), adaptee.getComparator());
- }
-
- @Override
- public SegmentScanner getSegmentScanner(long readPoint) {
- return adaptee.getSegmentScanner(readPoint);
- }
-
- @Override
- public boolean isEmpty() {
- return adaptee.isEmpty();
- }
-
- @Override
- public int getCellsCount() {
- return adaptee.getCellsCount();
- }
-
- @Override
- public long add(Cell cell) {
- return adaptee.add(cell);
- }
-
- @Override
- public Cell getFirstAfter(Cell cell) {
- return adaptee.getFirstAfter(cell);
- }
-
- @Override
- public void close() {
- adaptee.close();
- }
-
- @Override
- public Cell maybeCloneWithAllocator(Cell cell) {
- return adaptee.maybeCloneWithAllocator(cell);
- }
-
- @Override
- public Segment setSize(long size) {
- adaptee.setSize(size);
- return this;
- }
-
- @Override
- public long getSize() {
- return adaptee.getSize();
- }
-
- @Override
- public long rollback(Cell cell) {
- return adaptee.rollback(cell);
- }
-
- @Override
- public CellSet getCellSet() {
- return adaptee.getCellSet();
- }
-
- @Override
- public void dump(Log log) {
- adaptee.dump(log);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc449436/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegment.java
deleted file mode 100644
index 743416c..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegment.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import java.util.Iterator;
-import java.util.SortedSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparator;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
-/**
- * This mutable store segment encapsulates a mutable cell set and its respective memory allocation
- * buffers (MSLAB).
- */
-@InterfaceAudience.Private
-final class MutableCellSetSegment extends MutableSegment {
-
- private volatile CellSet cellSet;
- private final CellComparator comparator;
-
- // Instantiate objects only using factory
- MutableCellSetSegment(CellSet cellSet, MemStoreLAB memStoreLAB, long size,
- CellComparator comparator) {
- super(memStoreLAB, size);
- this.cellSet = cellSet;
- this.comparator = comparator;
- }
-
- @Override
- public SegmentScanner getSegmentScanner(long readPoint) {
- return new MutableCellSetSegmentScanner(this, readPoint);
- }
-
- @Override
- public boolean isEmpty() {
- return getCellSet().isEmpty();
- }
-
- @Override
- public int getCellsCount() {
- return getCellSet().size();
- }
-
- @Override
- public long add(Cell cell) {
- boolean succ = getCellSet().add(cell);
- long s = AbstractMemStore.heapSizeChange(cell, succ);
- updateMetaInfo(cell, s);
- // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call.
- // When we use ACL CP or Visibility CP which deals with Tags during
- // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not
- // parse the byte[] to identify the tags length.
- if(cell.getTagsLength() > 0) {
- tagsPresent = true;
- }
- return s;
- }
-
- @Override
- public long rollback(Cell cell) {
- Cell found = get(cell);
- if (found != null && found.getSequenceId() == cell.getSequenceId()) {
- long sz = AbstractMemStore.heapSizeChange(cell, true);
- remove(cell);
- incSize(-sz);
- return sz;
- }
- return 0;
- }
-
- @Override
- public Cell getFirstAfter(Cell cell) {
- SortedSet<Cell> snTailSet = tailSet(cell);
- if (!snTailSet.isEmpty()) {
- return snTailSet.first();
- }
- return null;
- }
-
- @Override
- public void dump(Log log) {
- for (Cell cell: getCellSet()) {
- log.debug(cell);
- }
- }
-
- @Override
- public SortedSet<Cell> tailSet(Cell firstCell) {
- return getCellSet().tailSet(firstCell);
- }
- @Override
- public CellSet getCellSet() {
- return cellSet;
- }
- @Override
- public CellComparator getComparator() {
- return comparator;
- }
-
- //*** Methods for MemStoreSegmentsScanner
- public Cell last() {
- return getCellSet().last();
- }
-
- public Iterator<Cell> iterator() {
- return getCellSet().iterator();
- }
-
- public SortedSet<Cell> headSet(Cell firstKeyOnRow) {
- return getCellSet().headSet(firstKeyOnRow);
- }
-
- public int compare(Cell left, Cell right) {
- return getComparator().compare(left, right);
- }
-
- public int compareRows(Cell left, Cell right) {
- return getComparator().compareRows(left, right);
- }
-
- private Cell get(Cell cell) {
- return getCellSet().get(cell);
- }
-
- private boolean remove(Cell e) {
- return getCellSet().remove(e);
- }
-
- // methods for tests
- @Override
- Cell first() {
- return this.getCellSet().first();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc449436/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegmentScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegmentScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegmentScanner.java
deleted file mode 100644
index 17791ff..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegmentScanner.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.SortedSet;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
-/**
- * A scanner of a single cells segment {@link MutableCellSetSegment}.
- */
-@InterfaceAudience.Private
-class MutableCellSetSegmentScanner extends SegmentScanner {
-
- // the observed structure
- private final MutableCellSetSegment segment;
- // the highest relevant MVCC
- private long readPoint;
- // the current iterator that can be reinitialized by
- // seek(), backwardSeek(), or reseek()
- private Iterator<Cell> iter;
- // the pre-calculated cell to be returned by peek()
- private Cell current = null;
- // or next()
- // A flag represents whether could stop skipping KeyValues for MVCC
- // if have encountered the next row. Only used for reversed scan
- private boolean stopSkippingKVsIfNextRow = false;
- // last iterated KVs by seek (to restore the iterator state after reseek)
- private Cell last = null;
-
- public MutableCellSetSegmentScanner(MutableCellSetSegment segment, long readPoint) {
- super();
- this.segment = segment;
- this.readPoint = readPoint;
- iter = segment.iterator();
- // the initialization of the current is required for working with heap of SegmentScanners
- current = getNext();
- //increase the reference count so the underlying structure will not be de-allocated
- this.segment.incScannerCount();
- }
-
- /**
- * Look at the next Cell in this scanner, but do not iterate the scanner
- * @return the currently observed Cell
- */
- @Override
- public Cell peek() { // sanity check, the current should be always valid
- if (current!=null && current.getSequenceId() > readPoint) {
- throw new RuntimeException("current is invalid: read point is "+readPoint+", " +
- "while current sequence id is " +current.getSequenceId());
- }
-
- return current;
- }
-
- /**
- * Return the next Cell in this scanner, iterating the scanner
- * @return the next Cell or null if end of scanner
- */
- @Override
- public Cell next() throws IOException {
- Cell oldCurrent = current;
- current = getNext(); // update the currently observed Cell
- return oldCurrent;
- }
-
- /**
- * Seek the scanner at or after the specified Cell.
- * @param cell seek value
- * @return true if scanner has values left, false if end of scanner
- */
- @Override
- public boolean seek(Cell cell) throws IOException {
- if(cell == null) {
- close();
- return false;
- }
- // restart the iterator from new key
- iter = segment.tailSet(cell).iterator();
- // last is going to be reinitialized in the next getNext() call
- last = null;
- current = getNext();
- return (current != null);
- }
-
- /**
- * Reseek the scanner at or after the specified KeyValue.
- * This method is guaranteed to seek at or after the required key only if the
- * key comes after the current position of the scanner. Should not be used
- * to seek to a key which may come before the current position.
- *
- * @param cell seek value (should be non-null)
- * @return true if scanner has values left, false if end of scanner
- */
- @Override
- public boolean reseek(Cell cell) throws IOException {
-
- /*
- See HBASE-4195 & HBASE-3855 & HBASE-6591 for the background on this implementation.
- This code is executed concurrently with flush and puts, without locks.
- The ideal implementation for performance would use the sub skip list implicitly
- pointed by the iterator. Unfortunately the Java API does not offer a method to
- get it. So we remember the last keys we iterated to and restore
- the reseeked set to at least that point.
- */
- iter = segment.tailSet(getHighest(cell, last)).iterator();
- current = getNext();
- return (current != null);
- }
-
- /**
- * Seek the scanner at or before the row of specified Cell, it firstly
- * tries to seek the scanner at or after the specified Cell, return if
- * peek KeyValue of scanner has the same row with specified Cell,
- * otherwise seek the scanner at the first Cell of the row which is the
- * previous row of specified KeyValue
- *
- * @param key seek Cell
- * @return true if the scanner is at the valid KeyValue, false if such Cell does not exist
- */
- @Override
- public boolean backwardSeek(Cell key) throws IOException {
- seek(key); // seek forward then go backward
- if (peek() == null || segment.compareRows(peek(), key) > 0) {
- return seekToPreviousRow(key);
- }
- return true;
- }
-
- /**
- * Seek the scanner at the first Cell of the row which is the previous row
- * of specified key
- *
- * @param cell seek value
- * @return true if the scanner at the first valid Cell of previous row,
- * false if not existing such Cell
- */
- @Override
- public boolean seekToPreviousRow(Cell cell) throws IOException {
- boolean keepSeeking = false;
- Cell key = cell;
-
- do {
- Cell firstKeyOnRow = CellUtil.createFirstOnRow(key);
- SortedSet<Cell> cellHead = segment.headSet(firstKeyOnRow);
- Cell lastCellBeforeRow = cellHead.isEmpty() ? null : cellHead.last();
- if (lastCellBeforeRow == null) {
- current = null;
- return false;
- }
- Cell firstKeyOnPreviousRow = CellUtil.createFirstOnRow(lastCellBeforeRow);
- this.stopSkippingKVsIfNextRow = true;
- seek(firstKeyOnPreviousRow);
- this.stopSkippingKVsIfNextRow = false;
- if (peek() == null
- || segment.getComparator().compareRows(peek(), firstKeyOnPreviousRow) > 0) {
- keepSeeking = true;
- key = firstKeyOnPreviousRow;
- continue;
- } else {
- keepSeeking = false;
- }
- } while (keepSeeking);
- return true;
- }
-
- /**
- * Seek the scanner at the first KeyValue of last row
- *
- * @return true if scanner has values left, false if the underlying data is empty
- */
- @Override
- public boolean seekToLastRow() throws IOException {
- Cell higherCell = segment.isEmpty() ? null : segment.last();
- if (higherCell == null) {
- return false;
- }
-
- Cell firstCellOnLastRow = CellUtil.createFirstOnRow(higherCell);
-
- if (seek(firstCellOnLastRow)) {
- return true;
- } else {
- return seekToPreviousRow(higherCell);
- }
- }
-
- @Override protected Segment getSegment() {
- return segment;
- }
-
- /********************* Private Methods **********************/
-
- /**
- * Private internal method for iterating over the segment,
- * skipping the cells with irrelevant MVCC
- */
- private Cell getNext() {
- Cell startKV = current;
- Cell next = null;
-
- try {
- while (iter.hasNext()) {
- next = iter.next();
- if (next.getSequenceId() <= this.readPoint) {
- return next; // skip irrelevant versions
- }
- if (stopSkippingKVsIfNextRow && // for backwardSeek() stay in the
- startKV != null && // boundaries of a single row
- segment.compareRows(next, startKV) > 0) {
- return null;
- }
- } // end of while
-
- return null; // nothing found
- } finally {
- if (next != null) {
- // in all cases, remember the last KV we iterated to, needed for reseek()
- last = next;
- }
- }
- }
-
- /**
- * Private internal method that returns the higher of the two key values, or null
- * if they are both null
- */
- private Cell getHighest(Cell first, Cell second) {
- if (first == null && second == null) {
- return null;
- }
- if (first != null && second != null) {
- int compare = segment.compare(first, second);
- return (compare > 0 ? first : second);
- }
- return (first != null ? first : second);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc449436/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
index fcaddb0..aef70e7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
@@ -18,34 +18,43 @@
*/
package org.apache.hadoop.hbase.regionserver;
-import java.util.SortedSet;
-
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
/**
- * An abstraction of a mutable segment in memstore, specifically the active segment.
+ * A mutable segment in memstore, specifically the active segment.
*/
@InterfaceAudience.Private
-public abstract class MutableSegment extends Segment {
+public class MutableSegment extends Segment {
- protected MutableSegment(MemStoreLAB memStoreLAB, long size) {
- super(memStoreLAB, size);
+ protected MutableSegment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB,
+ long size) {
+ super(cellSet, comparator, memStoreLAB, size);
}
/**
- * Returns a subset of the segment cell set, which starts with the given cell
- * @param firstCell a cell in the segment
- * @return a subset of the segment cell set, which starts with the given cell
+ * Adds the given cell into the segment
+ * @return the change in the heap size
*/
- public abstract SortedSet<Cell> tailSet(Cell firstCell);
+ public long add(Cell cell) {
+ return internalAdd(cell);
+ }
/**
- * Returns the Cell comparator used by this segment
- * @return the Cell comparator used by this segment
+ * Removes the given cell from the segment
+ * @return the change in the heap size
*/
- public abstract CellComparator getComparator();
+ public long rollback(Cell cell) {
+ Cell found = getCellSet().get(cell);
+ if (found != null && found.getSequenceId() == cell.getSequenceId()) {
+ long sz = AbstractMemStore.heapSizeChange(cell, true);
+ getCellSet().remove(cell);
+ incSize(-sz);
+ return sz;
+ }
+ return 0;
+ }
//methods for test
@@ -53,5 +62,7 @@ public abstract class MutableSegment extends Segment {
* Returns the first cell in the segment
* @return the first cell in the segment
*/
- abstract Cell first();
+ Cell first() {
+ return this.getCellSet().first();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc449436/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
index 7891809..135def9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
@@ -18,10 +18,13 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import java.util.Iterator;
+import java.util.SortedSet;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
@@ -38,12 +41,17 @@ import org.apache.hadoop.hbase.util.ByteRange;
@InterfaceAudience.Private
public abstract class Segment {
+ private volatile CellSet cellSet;
+ private final CellComparator comparator;
private volatile MemStoreLAB memStoreLAB;
private final AtomicLong size;
private final TimeRangeTracker timeRangeTracker;
protected volatile boolean tagsPresent;
- protected Segment(MemStoreLAB memStoreLAB, long size) {
+ protected Segment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, long
+ size) {
+ this.cellSet = cellSet;
+ this.comparator = comparator;
this.memStoreLAB = memStoreLAB;
this.size = new AtomicLong(size);
this.timeRangeTracker = new TimeRangeTracker();
@@ -51,6 +59,8 @@ public abstract class Segment {
}
protected Segment(Segment segment) {
+ this.cellSet = segment.getCellSet();
+ this.comparator = segment.getComparator();
this.memStoreLAB = segment.getMemStoreLAB();
this.size = new AtomicLong(segment.getSize());
this.timeRangeTracker = segment.getTimeRangeTracker();
@@ -58,46 +68,40 @@ public abstract class Segment {
}
/**
- * Creates the scanner that is able to scan the concrete segment
+ * Creates the scanner for the given read point
* @return a scanner for the given read point
*/
- public abstract SegmentScanner getSegmentScanner(long readPoint);
+ public SegmentScanner getSegmentScanner(long readPoint) {
+ return new SegmentScanner(this, readPoint);
+ }
/**
* Returns whether the segment has any cells
* @return whether the segment has any cells
*/
- public abstract boolean isEmpty();
+ public boolean isEmpty() {
+ return getCellSet().isEmpty();
+ }
/**
* Returns number of cells in segment
* @return number of cells in segment
*/
- public abstract int getCellsCount();
-
- /**
- * Adds the given cell into the segment
- * @return the change in the heap size
- */
- public abstract long add(Cell cell);
-
- /**
- * Removes the given cell from the segment
- * @return the change in the heap size
- */
- public abstract long rollback(Cell cell);
+ public int getCellsCount() {
+ return getCellSet().size();
+ }
/**
* Returns the first cell in the segment that has equal or greater key than the given cell
* @return the first cell in the segment that has equal or greater key than the given cell
*/
- public abstract Cell getFirstAfter(Cell cell);
-
- /**
- * Returns a set of all cells in the segment
- * @return a set of all cells in the segment
- */
- public abstract CellSet getCellSet();
+ public Cell getFirstAfter(Cell cell) {
+ SortedSet<Cell> snTailSet = tailSet(cell);
+ if (!snTailSet.isEmpty()) {
+ return snTailSet.first();
+ }
+ return null;
+ }
/**
* Closing a segment before it is being discarded
@@ -190,9 +194,69 @@ public abstract class Segment {
return timeRangeTracker;
}
+ //*** Methods for SegmentsScanner
+ public Cell last() {
+ return getCellSet().last();
+ }
+
+ public Iterator<Cell> iterator() {
+ return getCellSet().iterator();
+ }
+
+ public SortedSet<Cell> headSet(Cell firstKeyOnRow) {
+ return getCellSet().headSet(firstKeyOnRow);
+ }
+
+ public int compare(Cell left, Cell right) {
+ return getComparator().compare(left, right);
+ }
+
+ public int compareRows(Cell left, Cell right) {
+ return getComparator().compareRows(left, right);
+ }
+
+ /**
+ * Returns a set of all cells in the segment
+ * @return a set of all cells in the segment
+ */
+ protected CellSet getCellSet() {
+ return cellSet;
+ }
+
+ /**
+ * Returns the Cell comparator used by this segment
+ * @return the Cell comparator used by this segment
+ */
+ protected CellComparator getComparator() {
+ return comparator;
+ }
+
+ protected long internalAdd(Cell cell) {
+ boolean succ = getCellSet().add(cell);
+ long s = AbstractMemStore.heapSizeChange(cell, succ);
+ updateMetaInfo(cell, s);
+ return s;
+ }
+
protected void updateMetaInfo(Cell toAdd, long s) {
getTimeRangeTracker().includeTimestamp(toAdd);
size.addAndGet(s);
+ // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call.
+ // When we use ACL CP or Visibility CP which deals with Tags during
+ // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not
+ // parse the byte[] to identify the tags length.
+ if(toAdd.getTagsLength() > 0) {
+ tagsPresent = true;
+ }
+ }
+
+ /**
+ * Returns a subset of the segment cell set, which starts with the given cell
+ * @param firstCell a cell in the segment
+ * @return a subset of the segment cell set, which starts with the given cell
+ */
+ protected SortedSet<Cell> tailSet(Cell firstCell) {
+ return getCellSet().tailSet(firstCell);
}
private MemStoreLAB getMemStoreLAB() {
@@ -203,7 +267,11 @@ public abstract class Segment {
/**
* Dumps all cells of the segment into the given log
*/
- public abstract void dump(Log log);
+ void dump(Log log) {
+ for (Cell cell: getCellSet()) {
+ log.debug(cell);
+ }
+ }
@Override
public String toString() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc449436/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java
index ccb11df..394ffa1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java
@@ -54,7 +54,7 @@ public final class SegmentFactory {
}
public ImmutableSegment createImmutableSegment(final Configuration conf, MutableSegment segment) {
- return generateImmutableSegment(conf, segment);
+ return new ImmutableSegment(segment);
}
public MutableSegment createMutableSegment(final Configuration conf,
CellComparator comparator, long size) {
@@ -64,16 +64,11 @@ public final class SegmentFactory {
//****** private methods to instantiate concrete store segments **********//
- private ImmutableSegment generateImmutableSegment(final Configuration conf,
- MutableSegment segment) {
- // TBD use configuration to set type of segment
- return new ImmutableSegmentAdapter(segment);
- }
private MutableSegment generateMutableSegment(
final Configuration conf, CellComparator comparator, MemStoreLAB memStoreLAB, long size) {
// TBD use configuration to set type of segment
CellSet set = new CellSet(comparator);
- return new MutableCellSetSegment(set, memStoreLAB, size, comparator);
+ return new MutableSegment(set, comparator, memStoreLAB, size);
}
private MemStoreLAB getMemStoreLAB(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc449436/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java
index 8852d5c..b5aabb8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java
@@ -19,20 +19,193 @@
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
+import java.util.Iterator;
+import java.util.SortedSet;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Scan;
/**
- * An abstraction for store segment scanner.
+ * A scanner of a single memstore segment.
*/
@InterfaceAudience.Private
-public abstract class SegmentScanner implements KeyValueScanner {
+public class SegmentScanner implements KeyValueScanner {
private long sequenceID = Long.MAX_VALUE;
- protected abstract Segment getSegment();
+ // the observed structure
+ private final Segment segment;
+ // the highest relevant MVCC
+ private long readPoint;
+ // the current iterator that can be reinitialized by
+ // seek(), backwardSeek(), or reseek()
+ private Iterator<Cell> iter;
+ // the pre-calculated cell to be returned by peek()
+ private Cell current = null;
+ // or next()
+ // A flag represents whether could stop skipping KeyValues for MVCC
+ // if have encountered the next row. Only used for reversed scan
+ private boolean stopSkippingKVsIfNextRow = false;
+ // last iterated KVs by seek (to restore the iterator state after reseek)
+ private Cell last = null;
+
+ protected SegmentScanner(Segment segment, long readPoint) {
+ this.segment = segment;
+ this.readPoint = readPoint;
+ iter = segment.iterator();
+ // the initialization of the current is required for working with heap of SegmentScanners
+ current = getNext();
+ //increase the reference count so the underlying structure will not be de-allocated
+ this.segment.incScannerCount();
+ }
+
+ /**
+ * Look at the next Cell in this scanner, but do not iterate the scanner
+ * @return the currently observed Cell
+ */
+ @Override
+ public Cell peek() { // sanity check, the current should be always valid
+ if (current!=null && current.getSequenceId() > readPoint) {
+ throw new RuntimeException("current is invalid: read point is "+readPoint+", " +
+ "while current sequence id is " +current.getSequenceId());
+ }
+
+ return current;
+ }
+
+ /**
+ * Return the next Cell in this scanner, iterating the scanner
+ * @return the next Cell or null if end of scanner
+ */
+ @Override
+ public Cell next() throws IOException {
+ Cell oldCurrent = current;
+ current = getNext(); // update the currently observed Cell
+ return oldCurrent;
+ }
+
+ /**
+ * Seek the scanner at or after the specified Cell.
+ * @param cell seek value
+ * @return true if scanner has values left, false if end of scanner
+ */
+ @Override
+ public boolean seek(Cell cell) throws IOException {
+ if(cell == null) {
+ close();
+ return false;
+ }
+ // restart the iterator from new key
+ iter = segment.tailSet(cell).iterator();
+ // last is going to be reinitialized in the next getNext() call
+ last = null;
+ current = getNext();
+ return (current != null);
+ }
+
+ /**
+ * Reseek the scanner at or after the specified KeyValue.
+ * This method is guaranteed to seek at or after the required key only if the
+ * key comes after the current position of the scanner. Should not be used
+ * to seek to a key which may come before the current position.
+ *
+ * @param cell seek value (should be non-null)
+ * @return true if scanner has values left, false if end of scanner
+ */
+ @Override
+ public boolean reseek(Cell cell) throws IOException {
+
+ /*
+ See HBASE-4195 & HBASE-3855 & HBASE-6591 for the background on this implementation.
+ This code is executed concurrently with flush and puts, without locks.
+ The ideal implementation for performance would use the sub skip list implicitly
+ pointed by the iterator. Unfortunately the Java API does not offer a method to
+ get it. So we remember the last keys we iterated to and restore
+ the reseeked set to at least that point.
+ */
+ iter = segment.tailSet(getHighest(cell, last)).iterator();
+ current = getNext();
+ return (current != null);
+ }
+
+ /**
+ * Seek the scanner at or before the row of specified Cell, it firstly
+ * tries to seek the scanner at or after the specified Cell, return if
+ * peek KeyValue of scanner has the same row with specified Cell,
+ * otherwise seek the scanner at the first Cell of the row which is the
+ * previous row of specified KeyValue
+ *
+ * @param key seek Cell
+ * @return true if the scanner is at the valid KeyValue, false if such Cell does not exist
+ */
+ @Override
+ public boolean backwardSeek(Cell key) throws IOException {
+ seek(key); // seek forward then go backward
+ if (peek() == null || segment.compareRows(peek(), key) > 0) {
+ return seekToPreviousRow(key);
+ }
+ return true;
+ }
+
+ /**
+ * Seek the scanner at the first Cell of the row which is the previous row
+ * of specified key
+ *
+ * @param cell seek value
+ * @return true if the scanner at the first valid Cell of previous row,
+ * false if not existing such Cell
+ */
+ @Override
+ public boolean seekToPreviousRow(Cell cell) throws IOException {
+ boolean keepSeeking = false;
+ Cell key = cell;
+
+ do {
+ Cell firstKeyOnRow = CellUtil.createFirstOnRow(key);
+ SortedSet<Cell> cellHead = segment.headSet(firstKeyOnRow);
+ Cell lastCellBeforeRow = cellHead.isEmpty() ? null : cellHead.last();
+ if (lastCellBeforeRow == null) {
+ current = null;
+ return false;
+ }
+ Cell firstKeyOnPreviousRow = CellUtil.createFirstOnRow(lastCellBeforeRow);
+ this.stopSkippingKVsIfNextRow = true;
+ seek(firstKeyOnPreviousRow);
+ this.stopSkippingKVsIfNextRow = false;
+ if (peek() == null
+ || segment.getComparator().compareRows(peek(), firstKeyOnPreviousRow) > 0) {
+ keepSeeking = true;
+ key = firstKeyOnPreviousRow;
+ continue;
+ } else {
+ keepSeeking = false;
+ }
+ } while (keepSeeking);
+ return true;
+ }
+
+ /**
+ * Seek the scanner at the first KeyValue of last row
+ *
+ * @return true if scanner has values left, false if the underlying data is empty
+ */
+ @Override
+ public boolean seekToLastRow() throws IOException {
+ Cell higherCell = segment.isEmpty() ? null : segment.last();
+ if (higherCell == null) {
+ return false;
+ }
+
+ Cell firstCellOnLastRow = CellUtil.createFirstOnRow(higherCell);
+
+ if (seek(firstCellOnLastRow)) {
+ return true;
+ } else {
+ return seekToPreviousRow(higherCell);
+ }
+ }
/**
* Get the sequence id associated with this KeyValueScanner. This is required
@@ -140,6 +313,10 @@ public abstract class SegmentScanner implements KeyValueScanner {
return getSegment().shouldSeek(scan,oldestUnexpiredTS);
}
+ protected Segment getSegment(){
+ return segment;
+ }
+
//debug method
@Override
public String toString() {
@@ -149,4 +326,51 @@ public abstract class SegmentScanner implements KeyValueScanner {
return res;
}
+ /********************* Private Methods **********************/
+
+ /**
+ * Private internal method for iterating over the segment,
+ * skipping the cells with irrelevant MVCC
+ */
+ private Cell getNext() {
+ Cell startKV = current;
+ Cell next = null;
+
+ try {
+ while (iter.hasNext()) {
+ next = iter.next();
+ if (next.getSequenceId() <= this.readPoint) {
+ return next; // skip irrelevant versions
+ }
+ if (stopSkippingKVsIfNextRow && // for backwardSeek() stay in the
+ startKV != null && // boundaries of a single row
+ segment.compareRows(next, startKV) > 0) {
+ return null;
+ }
+ } // end of while
+
+ return null; // nothing found
+ } finally {
+ if (next != null) {
+ // in all cases, remember the last KV we iterated to, needed for reseek()
+ last = next;
+ }
+ }
+ }
+
+ /**
+ * Private internal method that returns the higher of the two key values, or null
+ * if they are both null
+ */
+ private Cell getHighest(Cell first, Cell second) {
+ if (first == null && second == null) {
+ return null;
+ }
+ if (first != null && second != null) {
+ int compare = segment.compare(first, second);
+ return (compare > 0 ? first : second);
+ }
+ return (first != null ? first : second);
+ }
+
}