You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/08/26 16:08:55 UTC
[32/34] incubator-kylin git commit: KYLIN-956 take care of II part
KYLIN-956 take care of II part
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d4e9652b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d4e9652b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d4e9652b
Branch: refs/heads/topn
Commit: d4e9652b5a04f30e516c06f512906772b7d90339
Parents: 99adbc6
Author: honma <ho...@ebay.com>
Authored: Wed Aug 26 18:02:39 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Aug 26 22:03:12 2015 +0800
----------------------------------------------------------------------
.../hadoop/invertedindex/IICreateHTableJob.java | 31 ++++++++++++++++----
.../storage/hbase/steps/CreateHTableJob.java | 21 +++++++------
2 files changed, 36 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d4e9652b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
index 752c3be..258f519 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
@@ -40,7 +40,6 @@ import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.steps.HBaseConnection;
import org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI;
-import org.apache.kylin.storage.hbase.util.LZOSupportnessChecker;
/**
* @author George Song (ysong1)
@@ -68,11 +67,33 @@ public class IICreateHTableJob extends AbstractHadoopJob {
HColumnDescriptor cf = new HColumnDescriptor(IIDesc.HBASE_FAMILY);
cf.setMaxVersions(1);
- if (LZOSupportnessChecker.getSupportness()) {
- logger.info("hbase will use lzo to compress II data");
+ String hbaseDefaultCC = config.getHbaseDefaultCompressionCodec().toLowerCase();
+
+ switch (hbaseDefaultCC) {
+ case "snappy": {
+ logger.info("hbase will use snappy to compress data");
+ cf.setCompressionType(Compression.Algorithm.SNAPPY);
+ break;
+ }
+ case "lzo": {
+ logger.info("hbase will use lzo to compress data");
cf.setCompressionType(Compression.Algorithm.LZO);
- } else {
- logger.info("hbase will not use lzo to compress II data");
+ break;
+ }
+ case "gz":
+ case "gzip": {
+ logger.info("hbase will use gzip to compress data");
+ cf.setCompressionType(Compression.Algorithm.GZ);
+ break;
+ }
+ case "lz4": {
+ logger.info("hbase will use lz4 to compress data");
+ cf.setCompressionType(Compression.Algorithm.LZ4);
+ break;
+ }
+ default: {
+ logger.info("hbase will not user any compression codec to compress data");
+ }
}
cf.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d4e9652b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index 7980813..9a80ae1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -27,8 +27,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
+import javax.annotation.Nullable;
+
import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -64,11 +64,11 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import javax.annotation.Nullable;
-
/**
*/
public class CreateHTableJob extends AbstractHadoopJob {
@@ -109,7 +109,6 @@ public class CreateHTableJob extends AbstractHadoopJob {
byte[][] splitKeys;
if (statistics_enabled) {
-
final Map<Long, Long> cuboidSizeMap = getCubeRowCountMapFromCuboidStatistics(cubeSegment, kylinConfig, conf);
splitKeys = getSplitsFromCuboidStatistics(cuboidSizeMap, kylinConfig, cubeSegment);
} else {
@@ -159,7 +158,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
byte[][] retValue = rowkeyList.toArray(new byte[rowkeyList.size()][]);
return retValue.length == 0 ? null : retValue;
}
-
+
public static Map<Long, Long> getCubeRowCountMapFromCuboidStatistics(CubeSegment cubeSegment, KylinConfig kylinConfig, Configuration conf) throws IOException {
ResourceStore rs = ResourceStore.getStore(kylinConfig);
String fileKey = cubeSegment.getStatisticsResourcePath();
@@ -174,7 +173,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
IOUtils.closeStream(is);
IOUtils.closeStream(tempFileStream);
}
-
+
Map<Long, Long> cuboidSizeMap = Maps.newHashMap();
FileSystem fs = HadoopUtil.getFileSystem("file:///" + tempFile.getAbsolutePath());
SequenceFile.Reader reader = null;
@@ -204,7 +203,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
}
return cuboidSizeMap;
}
-
+
public static Map<Long, Long> getCubeRowCountMapFromCuboidStatistics(Map<Long, HyperLogLogPlusCounter> counterMap, final int samplingPercentage) throws IOException {
Preconditions.checkArgument(samplingPercentage > 0);
return Maps.transformValues(counterMap, new Function<HyperLogLogPlusCounter, Long>() {
@@ -215,7 +214,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
}
});
}
-
+
public static byte[][] getSplitsFromCuboidStatistics(final Map<Long, Long> cubeRowCountMap, KylinConfig kylinConfig, CubeSegment cubeSegment) throws IOException {
final CubeDesc cubeDesc = cubeSegment.getCubeDesc();
@@ -229,7 +228,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
logger.info("Rowkey column " + i + " length " + cubeSegment.getColumnLength(columnList.get(i)));
rowkeyColumnSize.add(cubeSegment.getColumnLength(columnList.get(i)));
}
-
+
DataModelDesc.RealizationCapacity cubeCapacity = cubeDesc.getModel().getCapacity();
int cut = kylinConfig.getHBaseRegionCut(cubeCapacity.toString());
@@ -250,7 +249,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
for (Long cuboidSize : cubeSizeMap.values()) {
totalSizeInM += cuboidSize;
}
-
+
int nRegion = Math.round((float) totalSizeInM / (cut * 1024L));
nRegion = Math.max(kylinConfig.getHBaseRegionCutMin(), nRegion);
nRegion = Math.min(kylinConfig.getHBaseRegionCutMax(), nRegion);