You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by la...@apache.org on 2017/05/27 04:47:24 UTC
phoenix git commit: PHOENIX-3884 Correct MutationState size
estimation.
Repository: phoenix
Updated Branches:
refs/heads/4.x-HBase-1.2 a701600c6 -> 15387425c
PHOENIX-3884 Correct MutationState size estimation.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/15387425
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/15387425
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/15387425
Branch: refs/heads/4.x-HBase-1.2
Commit: 15387425cf2bc66a9fd3c17533bff1d67b186062
Parents: a701600
Author: Lars Hofhansl <la...@apache.org>
Authored: Fri May 26 21:48:16 2017 -0700
Committer: Lars Hofhansl <la...@apache.org>
Committed: Fri May 26 21:48:16 2017 -0700
----------------------------------------------------------------------
.../UngroupedAggregateRegionObserver.java | 24 ++++++++------------
.../apache/phoenix/execute/MutationState.java | 3 ++-
.../org/apache/phoenix/util/KeyValueUtil.java | 17 ++++++++++++++
3 files changed, 28 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15387425/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 70e74e9..16f19fb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -58,7 +58,6 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
@@ -288,8 +287,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
return s;
}
- class MutationList extends ArrayList<Mutation> implements HeapSize {
- private long heapSize = 0l;
+ class MutationList extends ArrayList<Mutation> {
+ private long byteSize = 0l;
public MutationList() {
super();
}
@@ -302,23 +301,18 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
public boolean add(Mutation e) {
boolean r = super.add(e);
if (r) {
- incrementHeapSize(e.heapSize());
+ this.byteSize += KeyValueUtil.calculateMutationDiskSize(e);
}
return r;
}
- @Override
- public long heapSize() {
- return heapSize;
- }
-
- private void incrementHeapSize(long heapSize) {
- this.heapSize += heapSize;
+ public long byteSize() {
+ return byteSize;
}
@Override
public void clear() {
- heapSize = 0l;
+ byteSize = 0l;
super.clear();
}
}
@@ -708,14 +702,14 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
}
}
- if (readyToCommit(rowCount, mutations.heapSize(), maxBatchSize, maxBatchSizeBytes)) {
+ if (readyToCommit(rowCount, mutations.byteSize(), maxBatchSize, maxBatchSizeBytes)) {
commit(region, mutations, indexUUID, blockingMemStoreSize, indexMaintainersPtr, txState,
areMutationInSameRegion, targetHTable, useIndexProto);
mutations.clear();
}
// Commit in batches based on UPSERT_BATCH_SIZE_BYTES_ATTRIB in config
- if (readyToCommit(rowCount, indexMutations.heapSize(), maxBatchSize, maxBatchSizeBytes)) {
+ if (readyToCommit(rowCount, indexMutations.byteSize(), maxBatchSize, maxBatchSizeBytes)) {
commitBatch(region, indexMutations, null, blockingMemStoreSize, null, txState,
useIndexProto);
indexMutations.clear();
@@ -894,7 +888,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
del.addDeleteMarker(cell);
}
}
- if (readyToCommit(rowCount, mutations.heapSize(), maxBatchSize, maxBatchSizeBytes)) {
+ if (readyToCommit(rowCount, mutations.byteSize(), maxBatchSize, maxBatchSizeBytes)) {
region.batchMutate(mutations.toArray(new Mutation[mutations.size()]), HConstants.NO_NONCE,
HConstants.NO_NONCE);
uuidValue = ServerCacheClient.generateId();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15387425/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
index 37ab7a6..d5cad63 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -87,6 +87,7 @@ import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.trace.util.Tracing;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.IndexUtil;
+import org.apache.phoenix.util.KeyValueUtil;
import org.apache.phoenix.util.LogUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.SQLCloseable;
@@ -837,7 +838,7 @@ public class MutationState implements SQLCloseable {
long byteSize = 0;
if (GlobalClientMetrics.isMetricsEnabled()) {
for (Mutation mutation : mutations) {
- byteSize += mutation.heapSize();
+ byteSize += KeyValueUtil.calculateMutationDiskSize(mutation);
}
}
GLOBAL_MUTATION_BYTES.update(byteSize);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15387425/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java
index 071de66..d16521b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java
@@ -21,11 +21,13 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.Map.Entry;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
@@ -160,4 +162,19 @@ public class KeyValueUtil {
return kvBuilder.compareQualifier(l, qualifier, 0, qualifier.length);
}
}
+
+ /**
+ * Calculate the size a mutation will likely take when stored in HBase
+ * @param m The Mutation
+ * @return the disk size of the passed mutation
+ */
+ public static long calculateMutationDiskSize(Mutation m) {
+ long size = 0;
+ for (Entry<byte [], List<Cell>> entry : m.getFamilyCellMap().entrySet()) {
+ for (Cell c : entry.getValue()) {
+ size += org.apache.hadoop.hbase.KeyValueUtil.length(c);
+ }
+ }
+ return size;
+ }
}