You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/05/04 09:54:34 UTC
[kylin] 01/01: CubeHFileJob should use currentHBaseConfiguration
but not new create hbase configuration
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch KYLIN-3168
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b9bcc92b87514a0d26c8ba9662db0735d9d3a3d5
Author: wyj <wu...@cmss.chinamobile.com>
AuthorDate: Tue Jan 16 19:38:52 2018 -0800
CubeHFileJob should use currentHBaseConfiguration but not new create hbase configuration
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../apache/kylin/storage/hbase/steps/CubeHFileJob.java | 15 +++++++--------
1 file changed, 7 insertions(+), 8 deletions(-)
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
index 093e8ee..ce4ceac 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
@@ -24,7 +24,6 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -74,11 +73,12 @@ public class CubeHFileJob extends AbstractHadoopJob {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cube = cubeMgr.getCube(cubeName);
- job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
+
+ // use current hbase configuration
+ Configuration configuration = HBaseConnection.getCurrentHBaseConfiguration();
+ job = Job.getInstance(configuration, getOptionValue(OPTION_JOB_NAME));
setJobClasspath(job, cube.getConfig());
- // For separate HBase cluster, note the output is a qualified HDFS path if "kylin.storage.hbase.cluster-fs" is configured, ref HBaseMRSteps.getHFilePath()
- HBaseConnection.addHBaseClusterNNHAConfiguration(job.getConfiguration());
addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
FileOutputFormat.setOutputPath(job, output);
@@ -88,12 +88,11 @@ public class CubeHFileJob extends AbstractHadoopJob {
// add metadata to distributed cache
attachCubeMetadata(cube, job.getConfiguration());
- Configuration hbaseConf = HBaseConfiguration.create(getConf());
- HTable htable = new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME));
+ HTable htable = new HTable(configuration, getOptionValue(OPTION_HTABLE_NAME));
// Automatic config !
HFileOutputFormat3.configureIncrementalLoad(job, htable);
- reconfigurePartitions(hbaseConf, partitionFilePath);
+ reconfigurePartitions(configuration, partitionFilePath);
job.setInputFormatClass(SequenceFileInputFormat.class);
job.setMapperClass(CubeHFileMapper.class);
@@ -103,7 +102,7 @@ public class CubeHFileJob extends AbstractHadoopJob {
job.setSortComparatorClass(RowKeyWritable.RowKeyComparator.class);
// set block replication to 3 for hfiles
- hbaseConf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "3");
+ configuration.set(DFSConfigKeys.DFS_REPLICATION_KEY, "3");
this.deletePath(job.getConfiguration(), output);
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.