You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2012/11/06 19:44:44 UTC
svn commit: r1406268 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/
main/java/org/apache/hadoop/hbase/regionserver/
main/java/org/apache/hadoop/hbase/regionserver/compactionhook/
test/java/org/apache/hadoop/hbase/regionserver/comp...
Author: liyin
Date: Tue Nov 6 18:44:43 2012
New Revision: 1406268
URL: http://svn.apache.org/viewvc?rev=1406268&view=rev
Log:
[HBASE-7099] Make a simpler interface used just for compact hooks in addition to the KeyValueAggregator
Author: adela
Summary: Simpler interface which should be used just for transforming values in KVs. The possible options for a KV would be to be skipped, transform the value, or leave the value untouched.
Test Plan: unit test
Reviewers: kannan, kranganathan, aaiyer, liyintang
Reviewed By: liyintang
CC: hbase-eng@, erling, liujiakai, jfan, ajaysomani
Differential Revision: https://phabricator.fb.com/D611834
Task ID: 1838741, 1749101
Added:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RestrictedKeyValue.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/compactionhook/
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/compactionhook/CompactionHook.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/LowerToUpperCompactionHook.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/SkipCompactionHook.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/TestCompactionHook.java
- copied, changed from r1405917, hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/LowerToUpperAggregator.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1406268&r1=1406267&r2=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Tue Nov 6 18:44:43 2012
@@ -595,11 +595,15 @@ public final class HConstants {
public static final int IPC_CALL_PARAMETER_LENGTH_MAX = 1000;
/**
- * used in Configuration to get/set the KV aggregator for compaction
- * hooks
+ * Used in Configuration to get/set the KV aggregator
*/
public static final String KV_AGGREGATOR = "kvaggregator";
+ /**
+ * Used in Configuration to get/set the compaction hook
+ */
+ public static final String COMPACTION_HOOK = "compaction_hook";
+
private HConstants() {
// Can't be instantiated with this ctor.
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1406268&r1=1406267&r2=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java Tue Nov 6 18:44:43 2012
@@ -967,6 +967,14 @@ public class KeyValue implements Writabl
}
/**
+ * @return return True if Put type.
+ */
+ public boolean isPut() {
+ int t = getType();
+ return (t == Type.Put.getCode()) ? true : false;
+ }
+
+ /**
* @return True if this KV is a {@link KeyValue.Type#Delete} type.
*/
public boolean isDeleteType() {
@@ -2188,4 +2196,14 @@ public class KeyValue implements Writabl
final int valueLength) {
return ROW_OFFSET + keyLength + valueLength;
}
+
+ /**
+ * Overwrites the value in the KV, keeps the other properties the same
+ */
+ public KeyValue modifyValueAndClone(byte[] newValue) {
+ KeyValue newKV = new KeyValue(this.getRow(), this.getFamily(),
+ this.getQualifier(), this.getTimestamp(), newValue);
+ newKV.setMemstoreTS(this.getMemstoreTS());
+ return newKV;
+ }
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1406268&r1=1406267&r2=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Nov 6 18:44:43 2012
@@ -343,7 +343,7 @@ public class HRegion implements HeapSize
}
oldVal.addAndGet(amount);
}
-
+
public static void clearNumericPersistentMetric(String key) {
numericPersistentMetrics.remove(key);
}
@@ -1175,7 +1175,7 @@ public class HRegion implements HeapSize
* @return split row if split is needed
* @throws IOException e
*/
- byte [] compactStores(final boolean majorCompaction)
+ public byte [] compactStores(final boolean majorCompaction)
throws IOException {
if (majorCompaction) {
this.triggerMajorCompaction();
@@ -1972,7 +1972,7 @@ public class HRegion implements HeapSize
* @param methodName "multiput_/multidelete_" to update metrics correctly.
* @throws IOException
*/
- public OperationStatusCode[] batchMutateWithLocks(Pair<Mutation, Integer>[] putsAndLocks,
+ public OperationStatusCode[] batchMutateWithLocks(Pair<Mutation, Integer>[] putsAndLocks,
String methodName) throws IOException {
this.writeRequests.incrTotalRequstCount();
BatchOperationInProgress<Pair<Mutation, Integer>> batchOp =
@@ -1998,7 +1998,7 @@ public class HRegion implements HeapSize
return batchOp.retCodes;
}
- private long doMiniBatchOp(BatchOperationInProgress<Pair<Mutation, Integer>> batchOp,
+ private long doMiniBatchOp(BatchOperationInProgress<Pair<Mutation, Integer>> batchOp,
String methodNameForMetricsUpdate) throws IOException {
String signature = null;
// variable to note if all Put items are for the same CF -- metrics related
@@ -2023,7 +2023,7 @@ public class HRegion implements HeapSize
byte[] previousRow = null;
Integer previousLockID = null;
Integer currentLockID = null;
-
+
while (lastIndexExclusive < batchOp.operations.length) {
Pair<Mutation, Integer> nextPair = batchOp.operations[lastIndexExclusive];
Mutation op = nextPair.getFirst();
@@ -2034,7 +2034,7 @@ public class HRegion implements HeapSize
checkFamilies(op.getFamilyMap().keySet());
checkTimestamps(op, now);
}
-
+
if (previousRow == null || !Bytes.equals(previousRow, op.getRow()) ||
(providedLockId != null && !previousLockID.equals(providedLockId))) {
// If we haven't got any rows in our batch, we should block to
@@ -2046,18 +2046,18 @@ public class HRegion implements HeapSize
assert !shouldBlock : "Should never fail to get lock when blocking";
break; // stop acquiring more rows for this batch
}
-
+
if (providedLockId == null) {
acquiredLocks.add(currentLockID);
}
-
+
// reset the previous row and lockID with the current one
previousRow = op.getRow();
previousLockID = currentLockID;
}
lastIndexExclusive++;
numReadyToWrite++;
-
+
// if first time around, designate expected signature for metric
// else, if all have been consistent so far, check if it still holds
// all else, designate failure signature and mark as unclear
@@ -2087,7 +2087,7 @@ public class HRegion implements HeapSize
// ----------------------------------
for (int i = firstIndex; i < lastIndexExclusive; i++) {
Mutation op = batchOp.operations[i].getFirst();
-
+
if (op instanceof Put) {
updateKVTimestamps(
op.getFamilyMap().values(),
@@ -2136,7 +2136,7 @@ public class HRegion implements HeapSize
if (locked) {
this.updatesLock.readLock().unlock();
}
-
+
releaseRowLocks(acquiredLocks);
// do after lock
@@ -2195,7 +2195,7 @@ public class HRegion implements HeapSize
try {
result = get(get);
boolean matches = false;
-
+
if (result.size() == 0) {
if (expectedValue == null ) {
matches = true;
@@ -2394,7 +2394,7 @@ public class HRegion implements HeapSize
private long applyFamilyMapToMemstore(Map<byte[], List<KeyValue>> familyMap) {
return applyFamilyMapToMemstore(familyMap, null);
}
-
+
private long applyFamilyMapToMemstore(Map<byte[], List<KeyValue>> familyMap,
MultiVersionConsistencyControl.WriteEntry writeEntryToUse) {
long start = EnvironmentEdgeManager.currentTimeMillis();
@@ -2896,7 +2896,7 @@ public class HRegion implements HeapSize
lockedRows.notifyAll();
}
}
-
+
/**
* Release the row locks!
* @param lockidList The list of the lock ID to release.
@@ -2969,7 +2969,7 @@ public class HRegion implements HeapSize
public void bulkLoadHFile(String hfilePath, byte[] familyName, boolean assignSeqId)
throws IOException {
long seqId = this.log.obtainSeqNum();
-
+
splitsAndClosesLock.readLock().lock();
try {
Store store = getStore(familyName);
@@ -2982,7 +2982,7 @@ public class HRegion implements HeapSize
splitsAndClosesLock.readLock().unlock();
}
}
-
+
public void bulkLoadHFile(String hfilePath, byte[] familyName)
throws IOException {
bulkLoadHFile(hfilePath, familyName, false);
@@ -3791,7 +3791,7 @@ public class HRegion implements HeapSize
boolean flush = false;
Integer lid = null;
-
+
splitsAndClosesLock.readLock().lock();
try {
// 1. run all pre-hooks before the atomic operation
@@ -3859,7 +3859,7 @@ public class HRegion implements HeapSize
}
// 10. run all coprocessor post hooks, after region lock is released
// NOT required in 0.89. coprocessors are not supported.
-
+
} finally {
if (lid != null) {
// 11. release the row lock
Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RestrictedKeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RestrictedKeyValue.java?rev=1406268&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RestrictedKeyValue.java (added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RestrictedKeyValue.java Tue Nov 6 18:44:43 2012
@@ -0,0 +1,76 @@
+
+/*
+ * Copyright The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+/**
+ * <p>
+ * This is a wrapper around the {@link KeyValue} to insure the user is disabled
+ * from modifying other fields in the KeyValue except the value. Please don't
+ * try to modify the visibility of keyValue from private to public, or generate
+ * a setter, you will introduce huge harms.
+ * </p>
+ *
+ *
+ * <p>
+ * We put this class in regionserver package since we want to be able to access
+ * the getKeyValue() method in {@link Store} and we want to restricted custom
+ * Compaction Hooks to access the method.
+ * </p>
+ */
+public class RestrictedKeyValue {
+ private KeyValue keyValue;
+
+ public RestrictedKeyValue(KeyValue keyValue) {
+ this.keyValue = keyValue;
+ }
+
+ /**
+ * If you don't call this method, the KV will stay unchanged.
+ * @param newValue
+ */
+ public void modifyValue(byte[] newValue) {
+ this.keyValue = keyValue.modifyValueAndClone(newValue);
+ }
+
+ public byte[] getValue() {
+ return keyValue.getValue();
+ }
+
+ /** don't change the modifier */
+ KeyValue getKeyValue() {
+ return keyValue;
+ }
+
+ public byte[] getRow() {
+ return keyValue.getRow();
+ }
+
+ public byte[] getFamily() {
+ return keyValue.getFamily();
+ }
+
+ public byte[] getQualifier() {
+ return keyValue.getQualifier();
+ }
+
+ public long getTimestamp() {
+ return keyValue.getTimestamp();
+ }
+}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1406268&r1=1406267&r2=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Tue Nov 6 18:44:43 2012
@@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.io.hfile.
import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.regionserver.compactionhook.CompactionHook;
import org.apache.hadoop.hbase.regionserver.kvaggregator.DefaultKeyValueAggregator;
import org.apache.hadoop.hbase.regionserver.kvaggregator.KeyValueAggregator;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
@@ -147,6 +148,7 @@ public class Store extends SchemaConfigu
final KeyValue.KVComparator comparator;
private Class<KeyValueAggregator> aggregatorClass = null;
+ private CompactionHook compactHook = null;
/**
* Constructor
@@ -251,6 +253,18 @@ public class Store extends SchemaConfigu
throw new IllegalArgumentException(e);
}
}
+ String compactHookString = conf.get(HConstants.COMPACTION_HOOK);
+ if (compactHookString != null && !compactHookString.isEmpty()) {
+ try {
+ this.compactHook = (CompactionHook) Class.forName(compactHookString).newInstance();
+ } catch (InstantiationException e) {
+ throw new IllegalArgumentException(e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalArgumentException(e);
+ } catch (ClassNotFoundException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
}
/**
@@ -435,7 +449,7 @@ public class Store extends SchemaConfigu
}
/**
- * Adds a value to the memstore
+ * Deletes a value to the memstore
*
* @param kv
* @return memstore size delta
@@ -1187,8 +1201,15 @@ public class Store extends SchemaConfigu
if (kv.getMemstoreTS() <= smallestReadPoint) {
kv.setMemstoreTS(0);
}
- writer.append(kv);
-
+ if (compactHook != null && kv.isPut()) {
+ RestrictedKeyValue restrictedKv = new RestrictedKeyValue(kv);
+ RestrictedKeyValue modifiedKv = compactHook.transform(restrictedKv);
+ if (modifiedKv != null) {
+ writer.append(modifiedKv.getKeyValue());
+ }
+ } else {
+ writer.append(kv);
+ }
// check periodically to see if a system stop is requested
if (Store.closeCheckInterval > 0) {
bytesWritten += kv.getLength();
@@ -1833,7 +1854,7 @@ public class Store extends SchemaConfigu
public static final long FIXED_OVERHEAD =
ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
- + (18 * ClassSize.REFERENCE) + (5 * Bytes.SIZEOF_LONG)
+ + (19 * ClassSize.REFERENCE) + (5 * Bytes.SIZEOF_LONG)
+ (4 * Bytes.SIZEOF_INT) + 2 * Bytes.SIZEOF_BOOLEAN) ;
public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/compactionhook/CompactionHook.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/compactionhook/CompactionHook.java?rev=1406268&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/compactionhook/CompactionHook.java (added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/compactionhook/CompactionHook.java Tue Nov 6 18:44:43 2012
@@ -0,0 +1,46 @@
+/**
+ * Copyright The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hbase.regionserver.compactionhook;
+
+import org.apache.hadoop.hbase.regionserver.RestrictedKeyValue;
+
+/**
+ * <p>
+ * This interface should be implemented for compaction hooks. The implementation
+ * should be stateless, because a common instance of the implementation might be
+ * reused across multiple active compactions.To see an example of a compaction
+ * hook implementation go here: {@link LowerToUpperCompactionHook}.
+ * </p>
+ */
+
+public interface CompactionHook {
+
+ /**
+ * This should be called to transform a KeyValue's value into a desired
+ * format. You can do the following transformations:
+ * <ul>
+ * <li>return null: that way you will skip this KV </li>
+ * <li>return the same kv: if you don't want to apply any changes to the KV</li>
+ * <li> transform only the value of the KV into the desired format</li>
+ * </ul>
+ *
+ * @param kv - the KeyValueForCompactionHooks that we want to transform
+ * @return the resulting KeyValueForCompactionHooks
+ */
+ RestrictedKeyValue transform(RestrictedKeyValue kv);
+
+}
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/LowerToUpperCompactionHook.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/LowerToUpperCompactionHook.java?rev=1406268&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/LowerToUpperCompactionHook.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/LowerToUpperCompactionHook.java Tue Nov 6 18:44:43 2012
@@ -0,0 +1,44 @@
+/**
+ * Copyright The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hbase.regionserver.compactionhook;
+
+import org.apache.hadoop.hbase.regionserver.RestrictedKeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * <p>
+ * Dummy compaction hook which transforms a lower-case value in KV to
+ * upper-case, implements CompactionHook interface.
+ * </p>
+ *
+ */
+public class LowerToUpperCompactionHook implements CompactionHook {
+
+ @Override
+ public RestrictedKeyValue transform (RestrictedKeyValue kv) {
+ byte[] newValue;
+ String currentValue = Bytes.toString(kv.getValue());
+ /**
+ * transform it to uppercase.
+ */
+ String newValueString = currentValue.toUpperCase();
+ newValue = Bytes.toBytes(newValueString);
+ kv.modifyValue(newValue);
+ return kv;
+ }
+}
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/SkipCompactionHook.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/SkipCompactionHook.java?rev=1406268&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/SkipCompactionHook.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/SkipCompactionHook.java Tue Nov 6 18:44:43 2012
@@ -0,0 +1,37 @@
+/**
+ * Copyright The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hbase.regionserver.compactionhook;
+
+import org.apache.hadoop.hbase.regionserver.RestrictedKeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class SkipCompactionHook implements CompactionHook {
+
+ /**
+ * Returns null only if the value of the KV contains 'c' or 'C' in its String
+ * representation
+ */
+ @Override
+ public RestrictedKeyValue transform(RestrictedKeyValue kv) {
+ String currentValue = Bytes.toString(kv.getValue());
+ if (currentValue.contains("C") || currentValue.contains("c")) {
+ return null;
+ }
+ return kv;
+ }
+}
Copied: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/TestCompactionHook.java (from r1405917, hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java)
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/TestCompactionHook.java?p2=hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/TestCompactionHook.java&p1=hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java&r1=1405917&r2=1406268&rev=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/TestCompactionHook.java Tue Nov 6 18:44:43 2012
@@ -1,4 +1,20 @@
-package org.apache.hadoop.hbase.regionserver.kvaggregator;
+/**
+ * Copyright The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hbase.regionserver.compactionhook;
import java.util.ArrayList;
import java.util.List;
@@ -21,8 +37,9 @@ import org.junit.BeforeClass;
import org.junit.Test;
@SuppressWarnings("deprecation")
-public class TestKeyValueAggregator {
- private static byte[] TABLE = Bytes.toBytes("TestKeyValueAggregator");
+public class TestCompactionHook {
+
+ private static byte[] TABLE = Bytes.toBytes("TestCompactionHook");
private static byte[] FAMILY = Bytes.toBytes("family");
private static byte[] START_KEY = Bytes.toBytes("aaa");
private static byte[] END_KEY = Bytes.toBytes("zzz");
@@ -42,12 +59,13 @@ public class TestKeyValueAggregator {
.setCompressionType(Compression.Algorithm.NONE));
TEST_UTIL
.getConfiguration()
- .set(HConstants.KV_AGGREGATOR,
- "org.apache.hadoop.hbase.regionserver.kvaggregator.LowerToUpperAggregator");
+ .set(
+ HConstants.COMPACTION_HOOK,
+ "org.apache.hadoop.hbase.regionserver.compactionhook.LowerToUpperCompactionHook");
}
@Test
- public void testDummyKvAggregator() throws Exception {
+ public void testLowerToUpperHook() throws Exception {
HRegion r = HBaseTestCase.createNewHRegion(TESTTABLEDESC, START_KEY,
END_KEY, TEST_UTIL.getConfiguration());
Put[] puts = new Put[25];
@@ -61,13 +79,19 @@ public class TestKeyValueAggregator {
puts[i] = put;
}
r.put(puts);
+ // without calling this, the compaction will not happen
+ r.flushcache();
+ r.compactStores(true);
+ //
Scan scan = new Scan();
InternalScanner s = r.getScanner(scan);
List<KeyValue> results = new ArrayList<KeyValue>();
while (s.next(results))
;
- // check if the values in results are in upper case
s.close();
+ // check if the values in results are in upper case
+ // this should return true, we are flushing the HRegion, so we expect the
+ // compaction to be done
Assert.assertTrue(checkIfLowerCase(results));
// check if we got all 25 results back
Assert.assertEquals(25, results.size());
@@ -83,4 +107,39 @@ public class TestKeyValueAggregator {
}
return true;
}
+
+ @Test
+ public void testSkipHook () throws Exception {
+ TEST_UTIL
+ .getConfiguration()
+ .set(
+ HConstants.COMPACTION_HOOK,
+ "org.apache.hadoop.hbase.regionserver.compactionhook.SkipCompactionHook");
+ HRegion r = HBaseTestCase.createNewHRegion(TESTTABLEDESC, START_KEY,
+ END_KEY, TEST_UTIL.getConfiguration());
+ Put[] puts = new Put[25];
+ // put some strings
+ for (int i = 0; i < 25; i++) {
+ byte[] row = Bytes.toBytes("row" + i);
+ Put put = new Put(row);
+ byte[] qualifier = Bytes.toBytes("qual" + i);
+ byte[] value = Bytes.toBytes("ab" + (char) (i + 97));
+ put.add(FAMILY, qualifier, value);
+ puts[i] = put;
+ }
+ r.put(puts);
+ // without calling this, the compaction will not happen
+ r.flushcache();
+ r.compactStores(true);
+ //
+ Scan scan = new Scan();
+ InternalScanner s = r.getScanner(scan);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ while (s.next(results))
+ ;
+ s.close();
+ // check if we got 24 results back, since "abc" should be skipped.
+ Assert.assertEquals(24, results.size());
+ }
+
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/LowerToUpperAggregator.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/LowerToUpperAggregator.java?rev=1406268&r1=1406267&r2=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/LowerToUpperAggregator.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/LowerToUpperAggregator.java Tue Nov 6 18:44:43 2012
@@ -1,7 +1,25 @@
+/**
+ * Copyright The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
package org.apache.hadoop.hbase.regionserver.kvaggregator;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
+import org.apache.hadoop.hbase.util.Bytes;
/**
* <p>
@@ -18,12 +36,12 @@ public class LowerToUpperAggregator impl
@Override
public KeyValue process(KeyValue kv) {
byte[] newValue;
- String currentValue = new String(kv.getValue());
+ String currentValue = Bytes.toString(kv.getValue());
/**
* transform it to uppercase.
*/
String newValueString = currentValue.toUpperCase();
- newValue = newValueString.getBytes();
+ newValue = Bytes.toBytes(newValueString);
KeyValue newKv = new KeyValue(kv.getRow(), kv.getFamily(),
kv.getQualifier(), kv.getTimestamp(), newValue);
return newKv;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java?rev=1406268&r1=1406267&r2=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java Tue Nov 6 18:44:43 2012
@@ -1,3 +1,20 @@
+/**
+ * Copyright The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
package org.apache.hadoop.hbase.regionserver.kvaggregator;
import java.util.ArrayList;