You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ma...@apache.org on 2016/02/24 22:13:21 UTC
[03/50] [abbrv] phoenix git commit: PHOENIX-2649 - GC/OOM during
BulkLoad (Sergey Soldatov)
PHOENIX-2649 - GC/OOM during BulkLoad (Sergey Soldatov)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a82a0ff6
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a82a0ff6
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a82a0ff6
Branch: refs/heads/calcite
Commit: a82a0ff608f90c470a1290296f8ed08243956fc9
Parents: 7972422
Author: ravimagham <ra...@bazaarvoice.com>
Authored: Thu Feb 4 14:38:43 2016 -0800
Committer: ravimagham <ra...@bazaarvoice.com>
Committed: Thu Feb 4 14:38:43 2016 -0800
----------------------------------------------------------------------
.../mapreduce/bulkload/TableRowkeyPair.java | 38 +++++++++++++++++---
.../mapreduce/bulkload/TestTableRowkeyPair.java | 6 ++--
2 files changed, 37 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a82a0ff6/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
index e3032be..ac80341 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
@@ -22,7 +22,7 @@ import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.io.WritableUtils;
@@ -30,6 +30,7 @@ import org.apache.hadoop.io.WritableUtils;
import com.google.common.base.Preconditions;
+
/**
* A WritableComparable to hold the table name and the rowkey.
*/
@@ -101,9 +102,38 @@ public class TableRowkeyPair implements WritableComparable<TableRowkeyPair> {
return this.tableName.compareTo(otherTableName);
}
}
-
- static {
- WritableComparator.define(TableRowkeyPair.class, new BytesWritable.Comparator());
+
+ /** Comparator for <code>TableRowkeyPair</code>. */
+ public static class Comparator extends WritableComparator {
+
+ public Comparator() {
+ super(TableRowkeyPair.class);
+ }
+
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ try {
+ // Compare table names
+ int strL1 = readInt(b1, s1);
+ int strL2 = readInt(b2, s2);
+ int cmp = compareBytes(b1, s1 + Bytes.SIZEOF_INT, strL1, b2, s2 + Bytes.SIZEOF_INT, strL2);
+ if (cmp != 0) {
+ return cmp;
+ }
+ // Compare row keys
+ int strL3 = readInt(b1, s1 + Bytes.SIZEOF_INT + strL1);
+ int strL4 = readInt(b2, s2 + Bytes.SIZEOF_INT + strL2);
+ int i = compareBytes(b1, s1 + Bytes.SIZEOF_INT*2 + strL1, strL3, b2, s2
+ + Bytes.SIZEOF_INT*2 + strL2, strL4);
+ return i;
+ } catch(Exception ex) {
+ throw new IllegalArgumentException(ex);
+ }
+ }
+ }
+
+ static {
+ WritableComparator.define(TableRowkeyPair.class, new Comparator());
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a82a0ff6/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java
index 1fee4bb..2a29c00 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.BytesWritable;
import org.junit.Assert;
import org.junit.Test;
@@ -38,8 +37,9 @@ public class TestTableRowkeyPair {
testsRowsKeys("first", "aa", "first", "ab", -1);
testsRowsKeys("second", "aa", "first", "aa", 1);
testsRowsKeys("first", "aa", "first", "aaa", -1);
+ testsRowsKeys("first","bb", "first", "aaaa", 1);
}
-
+
private void testsRowsKeys(String aTable, String akey, String bTable, String bkey, int expectedSignum) throws IOException {
final ImmutableBytesWritable arowkey = new ImmutableBytesWritable(Bytes.toBytes(akey));
@@ -48,7 +48,7 @@ public class TestTableRowkeyPair {
ImmutableBytesWritable browkey = new ImmutableBytesWritable(Bytes.toBytes(bkey));
TableRowkeyPair pair2 = new TableRowkeyPair(bTable, browkey);
- BytesWritable.Comparator comparator = new BytesWritable.Comparator();
+ TableRowkeyPair.Comparator comparator = new TableRowkeyPair.Comparator();
try( ByteArrayOutputStream baosA = new ByteArrayOutputStream();
ByteArrayOutputStream baosB = new ByteArrayOutputStream()) {