You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ct...@apache.org on 2016/04/24 02:50:39 UTC
hive git commit: HIVE-13527: Using deprecated APIs in HBase client
causes zookeeper connection leaks (Naveen Gangam via Chaoyu Tang)
Repository: hive
Updated Branches:
refs/heads/master 287f0451b -> 74197688f
HIVE-13527: Using deprecated APIs in HBase client causes zookeeper connection leaks (Naveen Gangam via Chaoyu Tang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/74197688
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/74197688
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/74197688
Branch: refs/heads/master
Commit: 74197688f18d9a6bee6393cfe1d7ab635beb3db5
Parents: 287f045
Author: ctang <ct...@cloudera.com>
Authored: Sat Apr 23 20:50:00 2016 -0400
Committer: ctang <ct...@cloudera.com>
Committed: Sat Apr 23 20:50:00 2016 -0400
----------------------------------------------------------------------
.../hive/hbase/HiveHBaseTableInputFormat.java | 33 ++++++++++++++++++--
1 file changed, 31 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/74197688/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
index 88d1865..d9db624 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
@@ -26,6 +26,9 @@ import java.util.Map;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
@@ -84,6 +87,7 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
static final Logger LOG = LoggerFactory.getLogger(HiveHBaseTableInputFormat.class);
private static final Object hbaseTableMonitor = new Object();
+ private Connection conn = null;
@Override
public RecordReader<ImmutableBytesWritable, ResultWritable> getRecordReader(
@@ -94,7 +98,11 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
HBaseSplit hbaseSplit = (HBaseSplit) split;
TableSplit tableSplit = hbaseSplit.getTableSplit();
- setHTable(HiveHBaseInputFormatUtil.getTable(jobConf));
+ if (conn == null) {
+ conn = ConnectionFactory.createConnection(HBaseConfiguration.create(jobConf));
+ }
+ initializeTable(conn, tableSplit.getTable());
+
setScan(HiveHBaseInputFormatUtil.getScan(jobConf));
Job job = new Job(jobConf);
@@ -107,6 +115,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
recordReader.initialize(tableSplit, tac);
} catch (InterruptedException e) {
closeTable(); // Free up the HTable connections
+ if (conn != null) {
+ conn.close();
+ conn = null;
+ }
throw new IOException("Failed to initialize RecordReader", e);
}
@@ -116,6 +128,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
public void close() throws IOException {
recordReader.close();
closeTable();
+ if (conn != null) {
+ conn.close();
+ conn = null;
+ }
}
@Override
@@ -442,7 +458,12 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
}
String hbaseTableName = jobConf.get(HBaseSerDe.HBASE_TABLE_NAME);
- setHTable(new HTable(HBaseConfiguration.create(jobConf), Bytes.toBytes(hbaseTableName)));
+ if (conn == null) {
+ conn = ConnectionFactory.createConnection(HBaseConfiguration.create(jobConf));
+ }
+ TableName tableName = TableName.valueOf(hbaseTableName);
+ initializeTable(conn, tableName);
+
String hbaseColumnsMapping = jobConf.get(HBaseSerDe.HBASE_COLUMNS_MAPPING);
boolean doColumnRegexMatching = jobConf.getBoolean(HBaseSerDe.HBASE_COLUMNS_REGEX_MATCHING, true);
@@ -509,6 +530,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
return results;
} finally {
closeTable();
+ if (conn != null) {
+ conn.close();
+ conn = null;
+ }
}
}
@@ -516,6 +541,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
protected void finalize() throws Throwable {
try {
closeTable();
+ if (conn != null) {
+ conn.close();
+ conn = null;
+ }
} finally {
super.finalize();
}