You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2014/05/01 19:15:49 UTC
git commit: CRUNCH-386: HBase 0.98.1 upgrade/fixes
Repository: crunch
Updated Branches:
refs/heads/master 2c9e8306c -> c88ce4718
CRUNCH-386: HBase 0.98.1 upgrade/fixes
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/c88ce471
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/c88ce471
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/c88ce471
Branch: refs/heads/master
Commit: c88ce47187b0a06ceba1e12ddee3077bf3f16b36
Parents: 2c9e830
Author: Josh Wills <jw...@apache.org>
Authored: Wed Apr 30 09:50:57 2014 -0700
Committer: Josh Wills <jw...@apache.org>
Committed: Thu May 1 09:56:42 2014 -0700
----------------------------------------------------------------------
.../org/apache/crunch/io/hbase/HFileSourceIT.java | 2 ++
.../org/apache/crunch/io/hbase/HFileTargetIT.java | 6 +++---
.../org/apache/crunch/io/hbase/HFileInputFormat.java | 2 +-
.../crunch/io/hbase/HFileOutputFormatForCrunch.java | 14 +++++++++-----
.../apache/crunch/io/hbase/HFileReaderFactory.java | 2 +-
.../java/org/apache/crunch/io/hbase/HFileUtils.java | 5 +----
pom.xml | 4 ++--
7 files changed, 19 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java
index 05c6a42..e82102b 100644
--- a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java
+++ b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileSourceIT.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Before;
import org.junit.Rule;
@@ -304,6 +305,7 @@ public class HFileSourceIT implements Serializable {
w = HFile.getWriterFactory(conf, new CacheConfig(conf))
.withPath(fs, inputPath)
.withComparator(KeyValue.COMPARATOR)
+ .withFileContext(new HFileContext())
.create();
for (KeyValue kv : sortedKVs) {
w.append(kv);
http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
index 7dd035e..25cec98 100644
--- a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
+++ b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
@@ -259,8 +259,8 @@ public class HFileTargetIT implements Serializable {
}
HFile.Reader reader = null;
try {
- reader = HFile.createReader(fs, f, new CacheConfig(conf));
- assertEquals(DataBlockEncoding.PREFIX, reader.getEncodingOnDisk());
+ reader = HFile.createReader(fs, f, new CacheConfig(conf), conf);
+ assertEquals(DataBlockEncoding.PREFIX, reader.getDataBlockEncoding());
} finally {
reader.close();
}
@@ -328,7 +328,7 @@ public class HFileTargetIT implements Serializable {
fs,
f,
new CacheConfig(fs.getConf()),
- DataBlockEncoding.NONE);
+ fs.getConf());
StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
scanner.seek(fakeKV); // have to call seek of each underlying scanner, otherwise KeyValueHeap won't work
scanners.add(scanner);
http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java
index 9ced0ac..43005ab 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileInputFormat.java
@@ -96,7 +96,7 @@ public class HFileInputFormat extends FileInputFormat<NullWritable, KeyValue> {
Path path = fileSplit.getPath();
FileSystem fs = path.getFileSystem(conf);
LOG.info("Initialize HFileRecordReader for " + path);
- this.in = HFile.createReader(fs, path, new CacheConfig(conf));
+ this.in = HFile.createReader(fs, path, new CacheConfig(conf), conf);
// The file info must be loaded before the scanner can be used.
// This seems like a bug in HBase, but it's easily worked around.
http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java
index 9d5f6ed..ad68da4 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileOutputFormatForCrunch.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -89,12 +90,8 @@ public class HFileOutputFormatForCrunch extends FileOutputFormat<Object, KeyValu
LOG.info("HColumnDescriptor: " + hcol.toString());
final HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
.withPath(fs, outputPath)
- .withBlockSize(hcol.getBlocksize())
- .withCompression(hcol.getCompression())
.withComparator(KeyValue.COMPARATOR)
- .withDataBlockEncoder(new HFileDataBlockEncoderImpl(hcol.getDataBlockEncoding()))
- .withChecksumType(HStore.getChecksumType(conf))
- .withBytesPerChecksum(HStore.getBytesPerChecksum(conf))
+ .withFileContext(getContext(hcol))
.create();
return new RecordWriter<Object, KeyValue>() {
@@ -124,4 +121,11 @@ public class HFileOutputFormatForCrunch extends FileOutputFormat<Object, KeyValu
}
};
}
+
+ private HFileContext getContext(HColumnDescriptor desc) {
+ HFileContext ctxt = new HFileContext();
+ ctxt.setDataBlockEncoding(desc.getDataBlockEncoding());
+ ctxt.setCompression(desc.getCompression());
+ return ctxt;
+ }
}
http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java
index f5db516..6189775 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileReaderFactory.java
@@ -39,7 +39,7 @@ public class HFileReaderFactory implements FileReaderFactory<KeyValue> {
Configuration conf = fs.getConf();
CacheConfig cacheConfig = new CacheConfig(conf);
try {
- HFile.Reader hfr = HFile.createReader(fs, path, cacheConfig);
+ HFile.Reader hfr = HFile.createReader(fs, path, cacheConfig, conf);
HFileScanner scanner = hfr.getScanner(
conf.getBoolean(HFILE_SCANNER_CACHE_BLOCKS, false),
conf.getBoolean(HFILE_SCANNER_PREAD, false));
http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java
index cb82dd8..63d2286 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java
@@ -86,7 +86,7 @@ public final class HFileUtils {
if ((cmp = compareType(l, r)) != 0) {
return cmp;
}
- return compareMemstoreTS(l, r);
+ return 0;
}
private int compareFamily(KeyValue l, KeyValue r) {
@@ -111,9 +111,6 @@ public final class HFileUtils {
return (int) r.getType() - (int) l.getType();
}
- private int compareMemstoreTS(KeyValue l, KeyValue r) {
- return Longs.compare(l.getMemstoreTS(), r.getMemstoreTS());
- }
};
private static class FilterByFamilyFn extends FilterFn<KeyValue> {
http://git-wip-us.apache.org/repos/asf/crunch/blob/c88ce471/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index aafe113..1787830 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,7 +88,7 @@ under the License.
<!-- Can be overridden by hadoop-2 profile, but these are the default values -->
<hadoop.version>1.1.2</hadoop.version>
- <hbase.version>0.96.0-hadoop1</hbase.version>
+ <hbase.version>0.98.1-hadoop1</hbase.version>
<hbase.midfix>hadoop1</hbase.midfix>
<scala.base.version>2.10</scala.base.version>
@@ -457,7 +457,7 @@ under the License.
</activation>
<properties>
<hadoop.version>2.2.0</hadoop.version>
- <hbase.version>0.96.0-hadoop2</hbase.version>
+ <hbase.version>0.98.1-hadoop2</hbase.version>
<commons-lang.version>2.5</commons-lang.version>
<hbase.midfix>hadoop2</hbase.midfix>
</properties>