You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by br...@apache.org on 2008/03/12 16:42:01 UTC
svn commit: r636373 - in /hadoop/hbase/trunk/src:
java/org/apache/hadoop/hbase/ java/org/apache/hadoop/hbase/client/
java/org/apache/hadoop/hbase/io/ java/org/apache/hadoop/hbase/ipc/
java/org/apache/hadoop/hbase/master/ java/org/apache/hadoop/hbase/re...
Author: bryanduxbury
Date: Wed Mar 12 08:41:54 2008
New Revision: 636373
URL: http://svn.apache.org/viewvc?rev=636373&view=rev
Log:
HBASE-430 Performance: Scanners and getRow return maps with duplicate data
-HRegionInterface's next method now returns RowResult objects
-All direct consumers of HRegionInterface make use of RowResults
Added:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/RowResult.java
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HMerge.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/TableOperation.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Writables.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/DisabledTestScanner2.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestGet.java
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HMerge.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HMerge.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HMerge.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HMerge.java Wed Mar 12 08:41:54 2008
@@ -335,8 +335,9 @@
root = new HRegion(rootTableDir, hlog, fs, conf,
HRegionInfo.rootRegionInfo, null, null);
- HScannerInterface rootScanner = root.getScanner(COL_REGIONINFO_ARRAY,
- new Text(), System.currentTimeMillis(), null);
+ HScannerInterface rootScanner =
+ root.getScanner(COL_REGIONINFO_ARRAY, new Text(),
+ HConstants.LATEST_TIMESTAMP, null);
try {
HStoreKey key = new HStoreKey();
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Wed Mar 12 08:41:54 2008
@@ -34,7 +34,6 @@
import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -43,6 +42,8 @@
import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.hbase.io.Cell;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
@@ -201,16 +202,15 @@
scannerId =
server.openScanner(firstMetaServer.getRegionInfo().getRegionName(),
COL_REGIONINFO_ARRAY, tableName, System.currentTimeMillis(), null);
- HbaseMapWritable values = server.next(scannerId);
+ RowResult values = server.next(scannerId);
if (values == null || values.size() == 0) {
break;
}
boolean found = false;
- for (Map.Entry<Writable, Writable> e: values.entrySet()) {
- HStoreKey key = (HStoreKey) e.getKey();
- if (key.getColumn().equals(COL_REGIONINFO)) {
+ for (Map.Entry<Text, Cell> e: values.entrySet()) {
+ if (e.getKey().equals(COL_REGIONINFO)) {
info = (HRegionInfo) Writables.getWritable(
- ((ImmutableBytesWritable) e.getValue()).get(), info);
+ e.getValue().getValue(), info);
if (info.getTableDesc().getName().equals(tableName)) {
found = true;
@@ -285,7 +285,7 @@
boolean isenabled = false;
while (true) {
- HbaseMapWritable values = server.next(scannerId);
+ RowResult values = server.next(scannerId);
if (values == null || values.size() == 0) {
if (valuesfound == 0) {
throw new NoSuchElementException(
@@ -294,11 +294,10 @@
break;
}
valuesfound += 1;
- for (Map.Entry<Writable, Writable> e: values.entrySet()) {
- HStoreKey key = (HStoreKey) e.getKey();
- if (key.getColumn().equals(COL_REGIONINFO)) {
+ for (Map.Entry<Text, Cell> e: values.entrySet()) {
+ if (e.getKey().equals(COL_REGIONINFO)) {
info = (HRegionInfo) Writables.getWritable(
- ((ImmutableBytesWritable) e.getValue()).get(), info);
+ e.getValue().getValue(), info);
isenabled = !info.isOffline();
break;
@@ -386,7 +385,7 @@
boolean disabled = false;
while (true) {
- HbaseMapWritable values = server.next(scannerId);
+ RowResult values = server.next(scannerId);
if (values == null || values.size() == 0) {
if (valuesfound == 0) {
throw new NoSuchElementException("table " + tableName + " not found");
@@ -394,11 +393,10 @@
break;
}
valuesfound += 1;
- for (Map.Entry<Writable, Writable> e: values.entrySet()) {
- HStoreKey key = (HStoreKey) e.getKey();
- if (key.getColumn().equals(COL_REGIONINFO)) {
+ for (Map.Entry<Text, Cell> e: values.entrySet()) {
+ if (e.getKey().equals(COL_REGIONINFO)) {
info = (HRegionInfo) Writables.getWritable(
- ((ImmutableBytesWritable) e.getValue()).get(), info);
+ e.getValue().getValue(), info);
disabled = info.isOffline();
break;
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java Wed Mar 12 08:41:54 2008
@@ -54,6 +54,7 @@
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.RowResult;
/**
* A non-instantiable class that manages connections to multiple tables in
@@ -265,27 +266,21 @@
// open a scanner over the meta region
scannerId = server.openScanner(
metaLocation.getRegionInfo().getRegionName(),
- COLUMN_FAMILY_ARRAY, startRow, LATEST_TIMESTAMP,
- null);
+ new Text[]{COL_REGIONINFO}, startRow, LATEST_TIMESTAMP, null);
// iterate through the scanner, accumulating unique table names
while (true) {
- HbaseMapWritable values = server.next(scannerId);
+ RowResult values = server.next(scannerId);
if (values == null || values.size() == 0) {
break;
}
- for (Map.Entry<Writable, Writable> e: values.entrySet()) {
- HStoreKey key = (HStoreKey) e.getKey();
- if (key.getColumn().equals(COL_REGIONINFO)) {
- HRegionInfo info = new HRegionInfo();
- info = (HRegionInfo) Writables.getWritable(
- ((ImmutableBytesWritable) e.getValue()).get(), info);
-
- // Only examine the rows where the startKey is zero length
- if (info.getStartKey().getLength() == 0) {
- uniqueTables.add(info.getTableDesc());
- }
- }
+
+ HRegionInfo info =
+ Writables.getHRegionInfo(values.get(COL_REGIONINFO));
+
+ // Only examine the rows where the startKey is zero length
+ if (info.getStartKey().getLength() == 0) {
+ uniqueTables.add(info.getTableDesc());
}
}
@@ -447,11 +442,9 @@
throw new IllegalStateException("region offline: " +
regionInfo.getRegionName());
}
-
- Cell serverValue = results.get(COL_SERVER);
- String serverAddress = Writables.bytesToString(
- serverValue == null ? null : serverValue.getValue());
+ String serverAddress =
+ Writables.cellToString(results.get(COL_SERVER));
if (serverAddress.equals("")) {
throw new NoServerForRegionException(
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java Wed Mar 12 08:41:54 2008
@@ -38,6 +38,7 @@
import org.apache.hadoop.hbase.filter.WhileMatchRowFilter;
import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.io.HbaseMapWritable;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Writables;
@@ -198,36 +199,28 @@
// open a scanner over the meta region
scannerId = server.openScanner(
metaLocation.getRegionInfo().getRegionName(),
- COLUMN_FAMILY_ARRAY, tableName, LATEST_TIMESTAMP,
+ new Text[]{COL_REGIONINFO}, tableName, LATEST_TIMESTAMP,
null);
- // iterate through the scanner, accumulating unique table names
- SCANNER_LOOP: while (true) {
- HbaseMapWritable values = server.next(scannerId);
+ // iterate through the scanner, accumulating unique region names
+ while (true) {
+ RowResult values = server.next(scannerId);
if (values == null || values.size() == 0) {
break;
}
- for (Map.Entry<Writable, Writable> e: values.entrySet()) {
- HStoreKey key = (HStoreKey) e.getKey();
- if (key.getColumn().equals(COL_REGIONINFO)) {
- HRegionInfo info = new HRegionInfo();
- info = (HRegionInfo) Writables.getWritable(
- ((ImmutableBytesWritable) e.getValue()).get(), info);
-
- if (!info.getTableDesc().getName().equals(this.tableName)) {
- break SCANNER_LOOP;
- }
-
- if (info.isOffline()) {
- continue SCANNER_LOOP;
- }
-
- if (info.isSplit()) {
- continue SCANNER_LOOP;
- }
-
- keyList.add(info.getStartKey());
- }
+
+ HRegionInfo info = new HRegionInfo();
+ info = (HRegionInfo) Writables.getWritable(
+ values.get(COL_REGIONINFO).getValue(), info);
+
+ if (!info.getTableDesc().getName().equals(this.tableName)) {
+ break;
+ }
+
+ if (info.isOffline() || info.isSplit()) {
+ continue;
+ } else {
+ keyList.add(info.getStartKey());
}
}
@@ -889,7 +882,7 @@
if (this.closed) {
return false;
}
- HbaseMapWritable values = null;
+ RowResult values = null;
// Clear the results so we don't inherit any values from any previous
// calls to next.
results.clear();
@@ -898,13 +891,12 @@
} while (values != null && values.size() == 0 && nextScanner());
if (values != null && values.size() != 0) {
- for (Map.Entry<Writable, Writable> e: values.entrySet()) {
- HStoreKey k = (HStoreKey) e.getKey();
- key.setRow(k.getRow());
- key.setVersion(k.getTimestamp());
+ for (Map.Entry<Text, Cell> e: values.entrySet()) {
+ // HStoreKey k = (HStoreKey) e.getKey();
+ key.setRow(values.getRow());
+ key.setVersion(e.getValue().getTimestamp());
key.setColumn(EMPTY_COLUMN);
- results.put(k.getColumn(),
- ((ImmutableBytesWritable) e.getValue()).get());
+ results.put(e.getKey(), e.getValue().getValue());
}
}
return values == null ? false : values.size() != 0;
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java Wed Mar 12 08:41:54 2008
@@ -44,6 +44,7 @@
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.RowResult;
/**
* This is a customized version of the polymorphic hadoop
@@ -120,6 +121,7 @@
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
+ addToMap(RowResult.class, code++);
}
private Class<?> declaredClass;
Added: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/RowResult.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/RowResult.java?rev=636373&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/RowResult.java (added)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/RowResult.java Wed Mar 12 08:41:54 2008
@@ -0,0 +1,168 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Collection;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+public class RowResult implements Writable, Map<Text, Cell> {
+ protected Text row;
+ protected HbaseMapWritable cells;
+
+ /**
+ * Used by Writable
+ */
+ public RowResult () {
+ row = new Text();
+ cells = new HbaseMapWritable();
+ }
+
+ /**
+ * Create a RowResult from a row and Cell map
+ */
+ public RowResult (final Text row, final HbaseMapWritable hbw) {
+ this.row = row;
+ this.cells = hbw;
+ }
+
+ /**
+ * Get the row for this RowResult
+ */
+ public Text getRow() {
+ return row;
+ }
+
+ //
+ // Map interface
+ //
+
+ public Cell put(Text key, Cell value) {
+ throw new UnsupportedOperationException("RowResult is read-only!");
+ }
+
+ public void putAll(Map map) {
+ throw new UnsupportedOperationException("RowResult is read-only!");
+ }
+
+ public Cell get(Object key) {
+ return (Cell)cells.get(key);
+ }
+
+ public Cell remove(Object key) {
+ throw new UnsupportedOperationException("RowResult is read-only!");
+ }
+
+ public boolean containsKey(Object key) {
+ return cells.containsKey(key);
+ }
+
+ public boolean containsValue(Object value) {
+ throw new UnsupportedOperationException("Don't support containsValue!");
+ }
+
+ public boolean isEmpty() {
+ return cells.isEmpty();
+ }
+
+ public int size() {
+ return cells.size();
+ }
+
+ public void clear() {
+ throw new UnsupportedOperationException("RowResult is read-only!");
+ }
+
+ public Set<Text> keySet() {
+ Set<Text> result = new HashSet<Text>();
+ for (Writable w : cells.keySet()) {
+ result.add((Text)w);
+ }
+ return result;
+ }
+
+ public Set<Map.Entry<Text, Cell>> entrySet() {
+ Set<Map.Entry<Text, Cell>> result = new HashSet<Map.Entry<Text, Cell>>();
+ for (Map.Entry<Writable, Writable> e : cells.entrySet()) {
+ result.add(new Entry((Text)e.getKey(), (Cell)e.getValue()));
+ }
+ return result;
+ }
+
+ public Collection<Cell> values() {
+ ArrayList<Cell> result = new ArrayList<Cell>();
+ for (Writable w : cells.values()) {
+ result.add((Cell)w);
+ }
+ return result;
+ }
+
+ /**
+ * Get the Cell that corresponds to column
+ */
+ public Cell get(Text column) {
+ return (Cell)cells.get(column);
+ }
+
+ public class Entry implements Map.Entry<Text, Cell> {
+ private Text row;
+ private Cell cell;
+
+ Entry(Text row, Cell cell) {
+ this.row = row;
+ this.cell = cell;
+ }
+
+ public Cell setValue(Cell c) {
+ throw new UnsupportedOperationException("RowResult is read-only!");
+ }
+
+ public Text getKey() {
+ return row;
+ }
+
+ public Cell getValue() {
+ return cell;
+ }
+ }
+
+ //
+ // Writable
+ //
+
+ public void readFields(final DataInput in) throws IOException {
+ row.readFields(in);
+ cells.readFields(in);
+ }
+
+ public void write(final DataOutput out) throws IOException {
+ row.write(out);
+ cells.write(out);
+ }
+}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java Wed Mar 12 08:41:54 2008
@@ -24,6 +24,7 @@
import org.apache.hadoop.hbase.filter.RowFilterInterface;
import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.io.HbaseMapWritable;
import org.apache.hadoop.io.Text;
@@ -219,7 +220,7 @@
* @return map of values
* @throws IOException
*/
- public HbaseMapWritable next(long scannerId) throws IOException;
+ public RowResult next(long scannerId) throws IOException;
/**
* Close a scanner
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java Wed Mar 12 08:41:54 2008
@@ -43,6 +43,7 @@
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
@@ -149,8 +150,8 @@
// Array to hold list of split parents found. Scan adds to list. After
// scan we go check if parents can be removed.
- Map<HRegionInfo, SortedMap<Text, byte[]>> splitParents =
- new HashMap<HRegionInfo, SortedMap<Text, byte[]>>();
+ Map<HRegionInfo, RowResult> splitParents =
+ new HashMap<HRegionInfo, RowResult>();
try {
regionServer = master.connection.getHRegionConnection(region.getServer());
scannerId =
@@ -159,22 +160,18 @@
int numberOfRegionsFound = 0;
while (true) {
- HbaseMapWritable values = regionServer.next(scannerId);
+ RowResult values = regionServer.next(scannerId);
if (values == null || values.size() == 0) {
break;
}
- // TODO: Why does this have to be a sorted map?
- SortedMap<Text, byte[]> results =
- RowMap.fromHbaseMapWritable(values).getMap();
-
- HRegionInfo info = master.getHRegionInfo(results);
+ HRegionInfo info = master.getHRegionInfo(values);
if (info == null) {
continue;
}
- String serverName = Writables.bytesToString(results.get(COL_SERVER));
- long startCode = Writables.bytesToLong(results.get(COL_STARTCODE));
+ String serverName = Writables.cellToString(values.get(COL_SERVER));
+ long startCode = Writables.cellToLong(values.get(COL_STARTCODE));
if (LOG.isDebugEnabled()) {
LOG.debug(Thread.currentThread().getName() + " regioninfo: {" +
info.toString() + "}, server: " + serverName + ", startCode: " +
@@ -184,7 +181,7 @@
// Note Region has been assigned.
checkAssigned(info, serverName, startCode);
if (isSplitParent(info)) {
- splitParents.put(info, results);
+ splitParents.put(info, values);
}
numberOfRegionsFound += 1;
}
@@ -216,8 +213,7 @@
// Scan is finished. Take a look at split parents to see if any we can
// clean up.
if (splitParents.size() > 0) {
- for (Map.Entry<HRegionInfo, SortedMap<Text, byte[]>> e:
- splitParents.entrySet()) {
+ for (Map.Entry<HRegionInfo, RowResult> e : splitParents.entrySet()) {
HRegionInfo hri = e.getKey();
cleanupSplits(region.getRegionName(), regionServer, hri, e.getValue());
}
@@ -252,8 +248,8 @@
* @throws IOException
*/
private boolean cleanupSplits(final Text metaRegionName,
- final HRegionInterface srvr, final HRegionInfo parent,
- SortedMap<Text, byte[]> rowContent)
+ final HRegionInterface srvr, final HRegionInfo parent,
+ RowResult rowContent)
throws IOException {
boolean result = false;
@@ -295,11 +291,11 @@
*/
protected boolean hasReferences(final Text metaRegionName,
final HRegionInterface srvr, final Text parent,
- SortedMap<Text, byte[]> rowContent, final Text splitColumn)
+ RowResult rowContent, final Text splitColumn)
throws IOException {
boolean result = false;
HRegionInfo split =
- Writables.getHRegionInfoOrNull(rowContent.get(splitColumn));
+ Writables.getHRegionInfoOrNull(rowContent.get(splitColumn).getValue());
if (split == null) {
return result;
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java Wed Mar 12 08:41:54 2008
@@ -46,6 +46,8 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.ipc.HbaseRPC;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.InfoServer;
@@ -614,18 +616,16 @@
long scannerid = srvr.openScanner(metaRegionName, COL_REGIONINFO_ARRAY,
tableName, System.currentTimeMillis(), null);
try {
- HbaseMapWritable data = srvr.next(scannerid);
+ RowResult data = srvr.next(scannerid);
// Test data and that the row for the data is for our table. If table
// does not exist, scanner will return row after where our table would
// be inserted if it exists so look for exact match on table name.
if (data != null && data.size() > 0) {
- for (Writable k: data.keySet()) {
- if (HRegionInfo.getTableNameFromRegionName(
- ((HStoreKey) k).getRow()).equals(tableName)) {
- // Then a region for this table already exists. Ergo table exists.
- throw new TableExistsException(tableName.toString());
- }
+ if (HRegionInfo.getTableNameFromRegionName(
+ data.getRow()).equals(tableName)) {
+ // Then a region for this table already exists. Ergo table exists.
+ throw new TableExistsException(tableName.toString());
}
}
} finally {
@@ -697,15 +697,15 @@
* @return Null or found HRegionInfo.
* @throws IOException
*/
- HRegionInfo getHRegionInfo(final Map<Text, byte[]> map)
+ HRegionInfo getHRegionInfo(final Map<Text, Cell> map)
throws IOException {
- byte [] bytes = map.get(COL_REGIONINFO);
- if (bytes == null) {
+ Cell regioninfo = map.get(COL_REGIONINFO);
+ if (regioninfo == null) {
LOG.warn(COL_REGIONINFO.toString() + " is empty; has keys: " +
map.keySet().toString());
return null;
}
- return (HRegionInfo)Writables.getWritable(bytes, new HRegionInfo());
+ return (HRegionInfo)Writables.getWritable(regioninfo.getValue(), new HRegionInfo());
}
/*
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java Wed Mar 12 08:41:54 2008
@@ -37,6 +37,7 @@
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.hbase.io.RowResult;
/**
* Instantiated when a server's lease has expired, meaning it has crashed.
@@ -97,7 +98,7 @@
try {
while (true) {
- HbaseMapWritable values = null;
+ RowResult values = null;
try {
values = server.next(scannerId);
} catch (IOException e) {
@@ -108,10 +109,9 @@
if (values == null || values.size() == 0) {
break;
}
- // TODO: Why does this have to be a sorted map?
- RowMap rm = RowMap.fromHbaseMapWritable(values);
- Text row = rm.getRow();
- SortedMap<Text, byte[]> map = rm.getMap();
+
+ Text row = values.getRow();
+
if (LOG.isDebugEnabled() && row != null) {
LOG.debug("shutdown scanner looking at " + row.toString());
}
@@ -121,7 +121,7 @@
// missed edits in hlog because hdfs does not do write-append).
String serverName;
try {
- serverName = Writables.bytesToString(map.get(COL_SERVER));
+ serverName = Writables.cellToString(values.get(COL_SERVER));
} catch (UnsupportedEncodingException e) {
LOG.error("Server name", e);
break;
@@ -133,7 +133,7 @@
}
// Bingo! Found it.
- HRegionInfo info = master.getHRegionInfo(map);
+ HRegionInfo info = master.getHRegionInfo(values);
if (info == null) {
continue;
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/TableOperation.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/TableOperation.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/TableOperation.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/TableOperation.java Wed Mar 12 08:41:54 2008
@@ -35,6 +35,8 @@
import org.apache.hadoop.hbase.io.HbaseMapWritable;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.RowResult;
/**
* Abstract base class for operations that need to examine all HRegionInfo
@@ -95,19 +97,19 @@
try {
while (true) {
- HbaseMapWritable values = server.next(scannerId);
+ RowResult values = server.next(scannerId);
if(values == null || values.size() == 0) {
break;
}
- RowMap rm = RowMap.fromHbaseMapWritable(values);
- SortedMap<Text, byte[]> map = rm.getMap();
- HRegionInfo info = this.master.getHRegionInfo(map);
+ HRegionInfo info = this.master.getHRegionInfo(values);
if (info == null) {
throw new IOException(COL_REGIONINFO + " not found on " +
- rm.getRow());
+ values.getRow());
}
- String serverName = Writables.bytesToString(map.get(COL_SERVER));
- long startCode = Writables.bytesToLong(map.get(COL_STARTCODE));
+ String serverName =
+ Writables.cellToString(values.get(COL_SERVER));
+ long startCode =
+ Writables.cellToLong(values.get(COL_STARTCODE));
if (info.getTableDesc().getName().compareTo(tableName) > 0) {
break; // Beyond any more entries for this table
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Wed Mar 12 08:41:54 2008
@@ -43,6 +43,7 @@
import org.apache.hadoop.hbase.io.BatchOperation;
import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
@@ -1204,7 +1205,8 @@
* @throws IOException
*/
public HScannerInterface getScanner(Text[] cols, Text firstRow,
- long timestamp, RowFilterInterface filter) throws IOException {
+ long timestamp, RowFilterInterface filter)
+ throws IOException {
lock.readLock().lock();
try {
if (this.closed.get()) {
@@ -1639,7 +1641,7 @@
/** Create an HScanner with a handle on many HStores. */
@SuppressWarnings("unchecked")
HScanner(Text[] cols, Text firstRow, long timestamp, HStore[] stores,
- RowFilterInterface filter)
+ RowFilterInterface filter)
throws IOException {
this.scanners = new HInternalScannerInterface[stores.length];
try {
@@ -1756,7 +1758,6 @@
return moreToFollow;
}
-
/** Shut down a single scanner */
void closeScanner(int i) {
try {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Wed Mar 12 08:41:54 2008
@@ -68,6 +68,7 @@
import org.apache.hadoop.hbase.filter.RowFilterInterface;
import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.io.HbaseMapWritable;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.ipc.HMasterRegionInterface;
@@ -1006,8 +1007,7 @@
}
/** {@inheritDoc} */
- public HbaseMapWritable next(final long scannerId) throws IOException {
-
+ public RowResult next(final long scannerId) throws IOException {
checkOpen();
requestCount.incrementAndGet();
try {
@@ -1024,8 +1024,7 @@
TreeMap<Text, byte []> results = new TreeMap<Text, byte []>();
while (s.next(key, results)) {
for(Map.Entry<Text, byte []> e: results.entrySet()) {
- values.put(new HStoreKey(key.getRow(), e.getKey(), key.getTimestamp()),
- new ImmutableBytesWritable(e.getValue()));
+ values.put(e.getKey(), new Cell(e.getValue(), key.getTimestamp()));
}
if(values.size() > 0) {
@@ -1036,8 +1035,7 @@
// No data for this row, go get another.
results.clear();
}
- return values;
-
+ return new RowResult(key.getRow(), values);
} catch (IOException e) {
checkFileSystem();
throw e;
@@ -1064,8 +1062,8 @@
/** {@inheritDoc} */
public long openScanner(Text regionName, Text[] cols, Text firstRow,
- final long timestamp, final RowFilterInterface filter)
- throws IOException {
+ final long timestamp, final RowFilterInterface filter)
+ throws IOException {
checkOpen();
requestCount.incrementAndGet();
try {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java Wed Mar 12 08:41:54 2008
@@ -64,7 +64,7 @@
import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.io.Cell;
-
+import org.apache.hadoop.hbase.io.RowResult;
/**
* HStore maintains a bunch of data files. It is responsible for maintaining
@@ -2252,7 +2252,8 @@
/** Create an Scanner with a handle on the memcache and HStore files. */
@SuppressWarnings("unchecked")
HStoreScanner(Text[] targetCols, Text firstRow, long timestamp,
- RowFilterInterface filter) throws IOException {
+ RowFilterInterface filter)
+ throws IOException {
this.dataFilter = filter;
if (null != dataFilter) {
@@ -2454,7 +2455,6 @@
return moreToFollow;
}
-
/** Shut down a single scanner */
void closeScanner(int i) {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Writables.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Writables.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Writables.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Writables.java Wed Mar 12 08:41:54 2008
@@ -29,6 +29,7 @@
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.hbase.io.Cell;
/**
* Utility class with methods for manipulating Writable objects
@@ -113,6 +114,19 @@
}
/**
+ * @param cell Cell object containing the serialized HRegionInfo
+ * @return A HRegionInfo instance built out of passed <code>cell</code>.
+ * @throws IOException
+ */
+ public static HRegionInfo getHRegionInfo(final Cell cell) throws IOException {
+ if (cell == null) {
+ return null;
+ } else {
+ return getHRegionInfo(cell.getValue());
+ }
+ }
+
+ /**
* Copy one Writable to another. Copies bytes using data streams.
* @param src Source Writable
* @param tgt Target Writable
@@ -183,5 +197,23 @@
return "";
}
return new String(bytes, HConstants.UTF8_ENCODING);
+ }
+
+ public static String cellToString(Cell c)
+ throws UnsupportedEncodingException {
+ if (c == null) {
+ return "";
+ } else {
+ return bytesToString(c.getValue());
+ }
+ }
+
+ public static long cellToLong(Cell c)
+ throws IOException {
+ if (c == null) {
+ return 0;
+ } else {
+ return bytesToLong(c.getValue());
+ }
}
}
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/DisabledTestScanner2.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/DisabledTestScanner2.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/DisabledTestScanner2.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/DisabledTestScanner2.java Wed Mar 12 08:41:54 2008
@@ -48,6 +48,7 @@
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.io.RowResult;
/**
* Additional scanner tests.
@@ -374,26 +375,19 @@
HConstants.COLUMN_FAMILY_ARRAY, new Text(),
System.currentTimeMillis(), null);
while (true) {
- TreeMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
- HbaseMapWritable values = regionServer.next(scannerId);
+ RowResult values = regionServer.next(scannerId);
if (values == null || values.size() == 0) {
break;
}
- for (Map.Entry<Writable, Writable> e: values.entrySet()) {
- HStoreKey k = (HStoreKey) e.getKey();
- results.put(k.getColumn(),
- ((ImmutableBytesWritable) e.getValue()).get());
- }
-
HRegionInfo info = (HRegionInfo) Writables.getWritable(
- results.get(HConstants.COL_REGIONINFO), new HRegionInfo());
+ values.get(HConstants.COL_REGIONINFO).getValue(), new HRegionInfo());
- byte[] bytes = results.get(HConstants.COL_SERVER);
- String serverName = Writables.bytesToString(bytes);
+ String serverName =
+ Writables.cellToString(values.get(HConstants.COL_SERVER));
long startCode =
- Writables.bytesToLong(results.get(HConstants.COL_STARTCODE));
+ Writables.cellToLong(values.get(HConstants.COL_STARTCODE));
LOG.info(Thread.currentThread().getName() + " scanner: "
+ Long.valueOf(scannerId) + ": regioninfo: {" + info.toString()
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestGet.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestGet.java?rev=636373&r1=636372&r2=636373&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestGet.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestGet.java Wed Mar 12 08:41:54 2008
@@ -67,7 +67,7 @@
for(Iterator<Text> i = values.keySet().iterator(); i.hasNext(); ) {
Text column = i.next();
if (column.equals(HConstants.COL_SERVER)) {
- String server = Writables.bytesToString(values.get(column).getValue());
+ String server = Writables.cellToString(values.get(column));
assertEquals(expectedServer, server);
LOG.info(server);
}