You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ra...@apache.org on 2016/02/03 23:20:06 UTC
[3/3] phoenix git commit: PHOENIX-2649 - GC/OOM during BulkLoad
PHOENIX-2649 - GC/OOM during BulkLoad
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d0174744
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d0174744
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d0174744
Branch: refs/heads/4.x-HBase-0.98
Commit: d0174744c5b8320169dcde89ce9a9106412c32e3
Parents: 91667bb
Author: ravimagham <ra...@bazaarvoice.com>
Authored: Wed Feb 3 14:17:26 2016 -0800
Committer: ravimagham <ra...@bazaarvoice.com>
Committed: Wed Feb 3 14:17:26 2016 -0800
----------------------------------------------------------------------
.../mapreduce/bulkload/TableRowkeyPair.java | 41 +++---------
.../mapreduce/bulkload/TestTableRowkeyPair.java | 67 ++++++++++++++++++++
2 files changed, 75 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d0174744/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
index 412226f..e3032be 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
@@ -84,6 +84,13 @@ public class TableRowkeyPair implements WritableComparable<TableRowkeyPair> {
WritableUtils.writeString(output,tableName);
rowkey.write(output);
}
+
+ @Override
+ public int hashCode() {
+ int result = this.tableName.hashCode();
+ result = 31 * result + this.rowkey.hashCode();
+ return result;
+ }
@Override
public int compareTo(TableRowkeyPair other) {
@@ -95,40 +102,8 @@ public class TableRowkeyPair implements WritableComparable<TableRowkeyPair> {
}
}
- /** Comparator optimized for <code>TableRowkeyPair</code>. */
- public static class Comparator extends WritableComparator {
- private BytesWritable.Comparator comparator = new BytesWritable.Comparator();
-
- public Comparator() {
- super(TableRowkeyPair.class);
- }
-
- @Override
- public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- try {
- int vintL1 = WritableUtils.decodeVIntSize(b1[s1]);
- int vintL2 = WritableUtils.decodeVIntSize(b2[s2]);
- int strL1 = readVInt(b1, s1);
- int strL2 = readVInt(b2, s2);
- int cmp = compareBytes(b1, s1 + vintL1, strL1, b2, s2 + vintL2, strL2);
- if (cmp != 0) {
- return cmp;
- }
- int vintL3 = WritableUtils.decodeVIntSize(b1[s1 + vintL1 + strL1]);
- int vintL4 = WritableUtils.decodeVIntSize(b2[s2 + vintL2 + strL2]);
- int strL3 = readVInt(b1, s1 + vintL1 + strL1);
- int strL4 = readVInt(b2, s2 + vintL2 + strL2);
- return comparator.compare(b1, s1 + vintL1 + strL1 + vintL3, strL3, b2, s2
- + vintL2 + strL2 + vintL4, strL4);
-
- } catch(Exception ex) {
- throw new IllegalArgumentException(ex);
- }
- }
- }
-
static {
- WritableComparator.define(TableRowkeyPair.class, new Comparator());
+ WritableComparator.define(TableRowkeyPair.class, new BytesWritable.Comparator());
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d0174744/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java
new file mode 100644
index 0000000..1fee4bb
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/bulkload/TestTableRowkeyPair.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce.bulkload;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.BytesWritable;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for {@linkplain TableRowkeyPair}
+ */
+public class TestTableRowkeyPair {
+
+ @Test
+ public void testRowkeyPair() throws IOException {
+ testsRowsKeys("first", "aa", "first", "aa", 0);
+ testsRowsKeys("first", "aa", "first", "ab", -1);
+ testsRowsKeys("second", "aa", "first", "aa", 1);
+ testsRowsKeys("first", "aa", "first", "aaa", -1);
+ }
+
+ private void testsRowsKeys(String aTable, String akey, String bTable, String bkey, int expectedSignum) throws IOException {
+
+ final ImmutableBytesWritable arowkey = new ImmutableBytesWritable(Bytes.toBytes(akey));
+ TableRowkeyPair pair1 = new TableRowkeyPair(aTable, arowkey);
+
+ ImmutableBytesWritable browkey = new ImmutableBytesWritable(Bytes.toBytes(bkey));
+ TableRowkeyPair pair2 = new TableRowkeyPair(bTable, browkey);
+
+ BytesWritable.Comparator comparator = new BytesWritable.Comparator();
+ try( ByteArrayOutputStream baosA = new ByteArrayOutputStream();
+ ByteArrayOutputStream baosB = new ByteArrayOutputStream()) {
+
+ pair1.write(new DataOutputStream(baosA));
+ pair2.write(new DataOutputStream(baosB));
+ Assert.assertEquals(expectedSignum , signum(pair1.compareTo(pair2)));
+ Assert.assertEquals(expectedSignum , signum(comparator.compare(baosA.toByteArray(), 0, baosA.size(), baosB.toByteArray(), 0, baosB.size())));
+ Assert.assertEquals(expectedSignum, -signum(comparator.compare(baosB.toByteArray(), 0, baosB.size(), baosA.toByteArray(), 0, baosA.size())));
+ }
+
+ }
+
+ private int signum(int i) {
+ return i > 0 ? 1: (i == 0 ? 0: -1);
+ }
+}