You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by nh...@apache.org on 2015/10/02 02:27:34 UTC
incubator-hawq git commit: HAWQ-5. Close HBase connection in the end
of PXF request
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-5 27a6043b5 -> 17b6c39a9
HAWQ-5. Close HBase connection in the end of PXF request
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/17b6c39a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/17b6c39a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/17b6c39a
Branch: refs/heads/HAWQ-5
Commit: 17b6c39a9964f27aa968c23ee17cefd86d217130
Parents: 27a6043
Author: Noa Horn <nh...@pivotal.io>
Authored: Thu Oct 1 17:27:21 2015 -0700
Committer: Noa Horn <nh...@pivotal.io>
Committed: Thu Oct 1 17:27:21 2015 -0700
----------------------------------------------------------------------
pxf/.gitignore | 1 +
.../pxf/plugins/hbase/HBaseAccessor.java | 26 +++++++++++---------
.../pxf/plugins/hbase/HBaseDataFragmenter.java | 4 ++-
.../hbase/utilities/HBaseLookupTable.java | 3 +--
.../plugins/hbase/utilities/HBaseUtilities.java | 17 +++++++++++++
.../com/pivotal/pxf/service/ReadBridge.java | 5 ++++
6 files changed, 41 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17b6c39a/pxf/.gitignore
----------------------------------------------------------------------
diff --git a/pxf/.gitignore b/pxf/.gitignore
index ce92f78..b5dbda3 100644
--- a/pxf/.gitignore
+++ b/pxf/.gitignore
@@ -13,3 +13,4 @@ docs
*.iml
.gradle
/log-regression
+.settings
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17b6c39a/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/HBaseAccessor.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/HBaseAccessor.java b/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/HBaseAccessor.java
index 88a5345..be6ba1f 100644
--- a/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/HBaseAccessor.java
+++ b/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/HBaseAccessor.java
@@ -6,6 +6,8 @@ import com.pivotal.pxf.api.utilities.InputData;
import com.pivotal.pxf.api.utilities.Plugin;
import com.pivotal.pxf.plugins.hbase.utilities.HBaseColumnDescriptor;
import com.pivotal.pxf.plugins.hbase.utilities.HBaseTupleDescription;
+import com.pivotal.pxf.plugins.hbase.utilities.HBaseUtilities;
+
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Connection;
@@ -66,9 +68,9 @@ public class HBaseAccessor extends Plugin implements ReadAccessor {
}
/**
- * Constructs {@link HBaseTupleDescription} based on HAWQ table description and
+ * Constructs {@link HBaseTupleDescription} based on HAWQ table description and
* initializes the scan start and end keys of the HBase table to default values.
- *
+ *
* @param input query information, contains HBase table name and filter
*/
public HBaseAccessor(InputData input) {
@@ -82,8 +84,8 @@ public class HBaseAccessor extends Plugin implements ReadAccessor {
/**
* Opens the HBase table.
- *
- * @return true if the current fragment (split) is
+ *
+ * @return true if the current fragment (split) is
* available for reading and includes in the filter
*/
@Override
@@ -102,7 +104,7 @@ public class HBaseAccessor extends Plugin implements ReadAccessor {
@Override
public void closeForRead() throws Exception {
table.close();
- connection.close();
+ HBaseUtilities.closeConnection(null, connection);
}
/**
@@ -130,10 +132,10 @@ public class HBaseAccessor extends Plugin implements ReadAccessor {
}
/**
- * Creates a {@link SplitBoundary} of the table split
- * this accessor instance is assigned to scan.
- * The table split is constructed from the fragment metadata
- * passed in {@link InputData#getFragmentMetadata()}.
+ * Creates a {@link SplitBoundary} of the table split
+ * this accessor instance is assigned to scan.
+ * The table split is constructed from the fragment metadata
+ * passed in {@link InputData#getFragmentMetadata()}.
* <p>
* The function verifies the split is within user supplied range.
* <p>
@@ -165,17 +167,17 @@ public class HBaseAccessor extends Plugin implements ReadAccessor {
* Returns true if given start/end key pair is within the scan range.
*/
private boolean withinScanRange(byte[] startKey, byte[] endKey) {
-
+
// startKey <= scanStartKey
if (Bytes.compareTo(startKey, scanStartKey) <= 0) {
// endKey == table's end or endKey >= scanStartKey
- if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) ||
+ if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) ||
Bytes.compareTo(endKey, scanStartKey) >= 0) {
return true;
}
} else { // startKey > scanStartKey
// scanEndKey == table's end or startKey <= scanEndKey
- if (Bytes.equals(scanEndKey, HConstants.EMPTY_END_ROW) ||
+ if (Bytes.equals(scanEndKey, HConstants.EMPTY_END_ROW) ||
Bytes.compareTo(startKey, scanEndKey) <= 0) {
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17b6c39a/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/HBaseDataFragmenter.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/HBaseDataFragmenter.java b/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/HBaseDataFragmenter.java
index c56d006..9ffdcc5 100644
--- a/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/HBaseDataFragmenter.java
+++ b/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/HBaseDataFragmenter.java
@@ -15,7 +15,6 @@ import java.io.IOException;
import java.io.ObjectOutputStream;
import java.util.List;
import java.util.Map;
-import java.util.NavigableMap;
/**
* Fragmenter class for HBase data resources.
@@ -54,12 +53,15 @@ public class HBaseDataFragmenter extends Fragmenter {
connection = ConnectionFactory.createConnection(hbaseConfiguration);
hbaseAdmin = connection.getAdmin();
if (!HBaseUtilities.isTableAvailable(hbaseAdmin, inputData.getDataSource())) {
+ HBaseUtilities.closeConnection(hbaseAdmin, connection);
throw new TableNotFoundException(inputData.getDataSource());
}
byte[] userData = prepareUserData();
addTableFragments(userData);
+ HBaseUtilities.closeConnection(hbaseAdmin, connection);
+
return fragments;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17b6c39a/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/utilities/HBaseLookupTable.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/utilities/HBaseLookupTable.java b/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/utilities/HBaseLookupTable.java
index 4b928b2..b522479 100644
--- a/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/utilities/HBaseLookupTable.java
+++ b/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/utilities/HBaseLookupTable.java
@@ -5,7 +5,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
@@ -167,7 +166,7 @@ public class HBaseLookupTable implements Closeable {
private void closeLookupTable() throws IOException {
lookupTable.close();
- connection.close();
+ HBaseUtilities.closeConnection(admin, connection);
}
private String lowerCase(byte[] key) {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17b6c39a/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/utilities/HBaseUtilities.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/utilities/HBaseUtilities.java b/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/utilities/HBaseUtilities.java
index c79518d..3edf07d 100644
--- a/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/utilities/HBaseUtilities.java
+++ b/pxf/pxf-hbase/src/main/java/com/pivotal/pxf/plugins/hbase/utilities/HBaseUtilities.java
@@ -6,6 +6,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
public class HBaseUtilities {
@@ -37,4 +38,20 @@ public class HBaseUtilities {
return hbaseAdmin.isTableAvailable(name) &&
hbaseAdmin.isTableEnabled(name);
}
+
+ /**
+ * Closes HBase admin and connection if they are open.
+ *
+ * @param hbaseAdmin
+ * @param hbaseConnection
+ * @throws IOException
+ */
+ public static void closeConnection(Admin hbaseAdmin, Connection hbaseConnection) throws IOException {
+ if (hbaseAdmin != null) {
+ hbaseAdmin.close();
+ }
+ if (hbaseConnection != null) {
+ hbaseConnection.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17b6c39a/pxf/pxf-service/src/main/java/com/pivotal/pxf/service/ReadBridge.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/com/pivotal/pxf/service/ReadBridge.java b/pxf/pxf-service/src/main/java/com/pivotal/pxf/service/ReadBridge.java
index 07268bf..f33972c 100644
--- a/pxf/pxf-service/src/main/java/com/pivotal/pxf/service/ReadBridge.java
+++ b/pxf/pxf-service/src/main/java/com/pivotal/pxf/service/ReadBridge.java
@@ -46,6 +46,7 @@ public class ReadBridge implements Bridge {
/*
* Accesses the underlying HDFS file
*/
+ @Override
public boolean beginIteration() throws Exception {
return fileAccessor.openForRead();
}
@@ -67,6 +68,7 @@ public class ReadBridge implements Bridge {
output = outputBuilder.makeOutput(fieldsResolver.getFields(onerow));
} catch (IOException ex) {
if (!isDataException(ex)) {
+ fileAccessor.closeForRead();
throw ex;
}
output = outputBuilder.getErrorOutput(ex);
@@ -81,6 +83,9 @@ public class ReadBridge implements Bridge {
Log.debug(ex.toString() + ": " + row_info);
}
output = outputBuilder.getErrorOutput(ex);
+ } catch (Exception ex) {
+ fileAccessor.closeForRead();
+ throw ex;
}
return output;