You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ss...@apache.org on 2017/02/20 01:38:18 UTC
[08/10] phoenix git commit: PHOENIX-3486 RoundRobinResultIterator
doesn't work correctly because of setting Scan's cache size inappropriately
in PhoenixInputForamt
PHOENIX-3486 RoundRobinResultIterator doesn't work correctly because of setting Scan's cache size inappropriately in PhoenixInputForamt
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1da595b0
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1da595b0
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1da595b0
Branch: refs/heads/4.x-HBase-1.0
Commit: 1da595b0364676174a0070a353eff6ad1e2ab706
Parents: b64b08f
Author: Jeongdae Kim <kj...@gmail.com>
Authored: Mon Nov 21 11:03:19 2016 +0900
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Sun Feb 19 17:21:58 2017 -0800
----------------------------------------------------------------------
.../hive/mapreduce/PhoenixInputFormat.java | 27 ++++++++++----------
1 file changed, 13 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1da595b0/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
index c5f6d18..3613b1f 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
@@ -23,6 +23,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HConnection;
@@ -144,11 +145,8 @@ public class PhoenixInputFormat<T extends DBWritable> implements InputFormat<Wri
.newJobContext(new Job(jobConf)));
boolean splitByStats = jobConf.getBoolean(PhoenixStorageHandlerConstants.SPLIT_BY_STATS,
false);
- int scanCacheSize = jobConf.getInt(PhoenixStorageHandlerConstants.HBASE_SCAN_CACHE, -1);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Generating splits with scanCacheSize : " + scanCacheSize);
- }
+ setScanCacheSize(jobConf);
// Adding Localization
HConnection connection = HConnectionManager.createConnection(jobConf);
@@ -168,10 +166,6 @@ public class PhoenixInputFormat<T extends DBWritable> implements InputFormat<Wri
if (splitByStats) {
for (Scan aScan : scans) {
- if (scanCacheSize > 0) {
- aScan.setCaching(scanCacheSize);
- }
-
if (LOG.isDebugEnabled()) {
LOG.debug("Split for scan : " + aScan + "with scanAttribute : " + aScan
.getAttributesMap() + " [scanCache, cacheBlock, scanBatch] : [" +
@@ -185,12 +179,6 @@ public class PhoenixInputFormat<T extends DBWritable> implements InputFormat<Wri
psplits.add(inputSplit);
}
} else {
- if (scanCacheSize > 0) {
- for (Scan aScan : scans) {
- aScan.setCaching(scanCacheSize);
- }
- }
-
if (LOG.isDebugEnabled()) {
LOG.debug("Scan count[" + scans.size() + "] : " + Bytes.toStringBinary(scans
.get(0).getStartRow()) + " ~ " + Bytes.toStringBinary(scans.get(scans
@@ -218,6 +206,17 @@ public class PhoenixInputFormat<T extends DBWritable> implements InputFormat<Wri
return psplits;
}
+ private void setScanCacheSize(JobConf jobConf) {
+ int scanCacheSize = jobConf.getInt(PhoenixStorageHandlerConstants.HBASE_SCAN_CACHE, -1);
+ if (scanCacheSize > 0) {
+ jobConf.setInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, scanCacheSize);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Generating splits with scanCacheSize : " + scanCacheSize);
+ }
+ }
+
@Override
public RecordReader<WritableComparable, T> getRecordReader(InputSplit split, JobConf job,
Reporter reporter) throws