You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/01/04 10:24:57 UTC
kylin git commit: KYLIN-2351 Support using non-default HDFS path as
kylin.env.hdfs-working-dir
Repository: kylin
Updated Branches:
refs/heads/yang22 ee04dca0d -> fa6cbe5bf
KYLIN-2351 Support using non-default HDFS path as kylin.env.hdfs-working-dir
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fa6cbe5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fa6cbe5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fa6cbe5b
Branch: refs/heads/yang22
Commit: fa6cbe5bf1d96e48f6cf88b020b954a63a2845c5
Parents: ee04dca
Author: Li Yang <li...@apache.org>
Authored: Wed Jan 4 18:24:48 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Jan 4 18:24:48 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfigBase.java | 9 +++++++++
.../java/org/apache/kylin/engine/mr/HadoopUtil.java | 15 ++++++++++++++-
.../kylin/engine/mr/common/AbstractHadoopJob.java | 4 ++--
.../kylin/engine/mr/steps/CreateDictionaryJob.java | 2 +-
.../engine/mr/steps/FactDistinctColumnsReducer.java | 10 ++++++----
.../kylin/engine/mr/steps/SaveStatisticsStep.java | 2 +-
.../apache/kylin/provision/BuildCubeWithEngine.java | 3 +--
.../org/apache/kylin/query/ITMassInQueryTest.java | 3 +--
.../apache/kylin/storage/hbase/ITHdfsOpsTest.java | 4 +---
.../org/apache/kylin/source/hive/HiveMRInput.java | 2 +-
.../org/apache/kylin/source/kafka/KafkaMRInput.java | 2 +-
.../kylin/storage/hbase/HBaseResourceStore.java | 11 ++++-------
.../kylin/storage/hbase/steps/DeprecatedGCStep.java | 3 +--
.../hbase/steps/HDFSPathGarbageCollectionStep.java | 2 +-
.../hbase/steps/SequenceFileCuboidWriter.java | 2 +-
.../kylin/storage/hbase/util/CubeMigrationCLI.java | 3 ++-
.../kylin/storage/hbase/util/StorageCleanupJob.java | 3 ++-
.../java/org/apache/kylin/tool/CubeMigrationCLI.java | 3 ++-
.../org/apache/kylin/tool/StorageCleanupJob.java | 4 ++--
19 files changed, 53 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 194f158..5b7c734 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -171,6 +171,15 @@ abstract public class KylinConfigBase implements Serializable {
if (!root.endsWith("/")) {
root += "/";
}
+
+ // make sure path qualified
+ if (!root.contains("://")) {
+ if (!root.startsWith("/"))
+ root = "hdfs:///" + root;
+ else
+ root = "hdfs://" + root;
+ }
+
return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 88692a0..3d29a02 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.Writable;
+import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,10 +67,22 @@ public class HadoopUtil {
return conf;
}
+ public static FileSystem getWorkingFileSystem() throws IOException {
+ return getFileSystem(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
+ }
+
+ public static FileSystem getWorkingFileSystem(Configuration conf) throws IOException {
+ return getFileSystem(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory(), conf);
+ }
+
public static FileSystem getFileSystem(String path) throws IOException {
- return FileSystem.get(makeURI(path), getCurrentConfiguration());
+ return getFileSystem(path, getCurrentConfiguration());
}
+ static FileSystem getFileSystem(String path, Configuration conf) throws IOException {
+ return FileSystem.get(makeURI(path), conf);
+ }
+
public static URI makeURI(String filePath) {
try {
return new URI(fixWindowsPath(filePath));
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 80636d3..243e59d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -284,7 +284,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
try {
Configuration jobConf = job.getConfiguration();
FileSystem localfs = FileSystem.getLocal(jobConf);
- FileSystem hdfs = FileSystem.get(jobConf);
+ FileSystem hdfs = HadoopUtil.getWorkingFileSystem(jobConf);
StringBuilder jarList = new StringBuilder();
StringBuilder fileList = new StringBuilder();
@@ -408,7 +408,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
inp = inp.trim();
if (inp.endsWith("/*")) {
inp = inp.substring(0, inp.length() - 2);
- FileSystem fs = FileSystem.get(job.getConfiguration());
+ FileSystem fs = HadoopUtil.getWorkingFileSystem(job.getConfiguration());
Path path = new Path(inp);
if (!exists(fs, path)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index 4985503..8187ff8 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -65,7 +65,7 @@ public class CreateDictionaryJob extends AbstractHadoopJob {
public Dictionary<String> getDictionary(TblColRef col) throws IOException {
Path colDir = new Path(factColumnsInputPath, col.getName());
Path dictFile = new Path(colDir, col.getName() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
- FileSystem fs = HadoopUtil.getFileSystem(dictFile.toString());
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
if (fs.exists(dictFile) == false)
return null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index a59e3ae..6ef976e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -43,6 +43,7 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.IDictionaryBuilder;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -199,7 +200,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private void outputDistinctValues(TblColRef col, Collection<ByteArray> values, Context context) throws IOException {
final Configuration conf = context.getConfiguration();
- final FileSystem fs = FileSystem.get(conf);
+ final FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH);
final Path colDir = new Path(outputPath, col.getName());
final String fileName = col.getName() + "-" + taskId % uhcReducerCount;
@@ -255,7 +256,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private FSDataOutputStream getOutputStream(Context context, String outputFileName) throws IOException {
final Configuration conf = context.getConfiguration();
- final FileSystem fs = FileSystem.get(conf);
+ final FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH);
final Path colDir = new Path(outputPath, col.getName());
final Path outputFile = new Path(colDir, outputFileName);
@@ -304,8 +305,9 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private void writeMapperAndCuboidStatistics(Context context) throws IOException {
Configuration conf = context.getConfiguration();
- FileSystem fs = FileSystem.get(conf);
- FSDataOutputStream out = fs.create(new Path(statisticsOutput, BatchConstants.CFG_STATISTICS_CUBE_ESTIMATION_FILENAME));
+ FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
+ Path path = new Path(statisticsOutput, BatchConstants.CFG_STATISTICS_CUBE_ESTIMATION_FILENAME);
+ FSDataOutputStream out = fs.create(path);
try {
String msg;
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 020c62c..6120270 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -59,8 +59,8 @@ public class SaveStatisticsStep extends AbstractExecutable {
ResourceStore rs = ResourceStore.getStore(kylinConf);
try {
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
Path statisticsFilePath = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams()), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
- FileSystem fs = FileSystem.get(HadoopUtil.getCurrentConfiguration());
if (!fs.exists(statisticsFilePath))
throw new IOException("File " + statisticsFilePath + " does not exists");
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 67b62d5..7a2954d 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -114,8 +114,7 @@ public class BuildCubeWithEngine {
try {
//check hdfs permission
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
+ FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
String hdfsWorkingDirectory = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
Path coprocessorDir = new Path(hdfsWorkingDirectory);
boolean success = fileSystem.mkdirs(coprocessorDir);
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
index 5835baf..18d79ae 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
@@ -68,8 +68,7 @@ public class ITMassInQueryTest extends KylinTestBase {
ITKylinQueryTest.joinType = "left";
ITKylinQueryTest.setupAll();
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
- fileSystem = FileSystem.get(hconf);
+ fileSystem = HadoopUtil.getWorkingFileSystem();
int sellerCount = 200;
Random r = new Random();
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
index 499a456..786d7d1 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
@@ -42,9 +42,7 @@ public class ITHdfsOpsTest extends HBaseMetadataTestCase {
this.createTestMetadata();
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
-
- fileSystem = FileSystem.get(hconf);
+ fileSystem = HadoopUtil.getWorkingFileSystem();
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index fc2b982..0f351b0 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -379,7 +379,7 @@ public class HiveMRInput implements IMRInput {
private void rmdirOnHDFS(String path) throws IOException {
Path externalDataPath = new Path(path);
- FileSystem fs = FileSystem.get(externalDataPath.toUri(), HadoopUtil.getCurrentConfiguration());
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
if (fs.exists(externalDataPath)) {
fs.delete(externalDataPath, true);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index dde3584..2f4fded 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -223,7 +223,7 @@ public class KafkaMRInput implements IMRInput {
private void rmdirOnHDFS(String path) throws IOException {
Path externalDataPath = new Path(path);
- FileSystem fs = FileSystem.get(externalDataPath.toUri(), HadoopUtil.getCurrentConfiguration());
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
if (fs.exists(externalDataPath)) {
fs.delete(externalDataPath, true);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 1d19983..2a12984 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -27,7 +27,6 @@ import java.util.NavigableSet;
import java.util.TreeSet;
import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -49,6 +48,7 @@ import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -201,8 +201,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] value = r.getValue(B_FAMILY, B_COLUMN);
if (value.length == 0) {
Path redirectPath = bigCellHDFSPath(resPath);
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
+ FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
return fileSystem.open(redirectPath);
} else {
@@ -292,8 +291,7 @@ public class HBaseResourceStore extends ResourceStore {
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
+ FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
if (fileSystem.exists(redirectPath)) {
fileSystem.delete(redirectPath, true);
@@ -339,8 +337,7 @@ public class HBaseResourceStore extends ResourceStore {
private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
+ FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
if (fileSystem.exists(redirectPath)) {
fileSystem.delete(redirectPath, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index 46a828e..fbe64d9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -132,8 +132,7 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldHdfsPaths = this.getOldHdsfPaths();
if (oldHdfsPaths != null && oldHdfsPaths.size() > 0) {
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
+ FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
for (String path : oldHdfsPaths) {
if (path.endsWith("*"))
path = path.substring(0, path.length() - 1);
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
index fbfd582..89baf95 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
@@ -59,7 +59,7 @@ public class HDFSPathGarbageCollectionStep extends AbstractExecutable {
try {
config = new JobEngineConfig(context.getConfig());
List<String> toDeletePaths = getDeletePaths();
- dropHdfsPathOnCluster(toDeletePaths, FileSystem.get(HadoopUtil.getCurrentConfiguration()));
+ dropHdfsPathOnCluster(toDeletePaths, HadoopUtil.getWorkingFileSystem());
if (StringUtils.isNotEmpty(context.getConfig().getHBaseClusterFs())) {
dropHdfsPathOnCluster(toDeletePaths, FileSystem.get(HBaseConnection.getCurrentHBaseConfiguration()));
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
index 8f2fc80..1d66d3e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
@@ -55,7 +55,7 @@ public class SequenceFileCuboidWriter extends KVGTRecordWriter {
JobBuilderSupport jobBuilderSupport = new JobBuilderSupport(cubeSegment, "SYSTEM");
String cuboidRoot = jobBuilderSupport.getCuboidRootPath(cubeSegment);
Path cuboidPath = new Path(cuboidRoot);
- FileSystem fs = HadoopUtil.getFileSystem(cuboidRoot);
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
try {
if (fs.exists(cuboidPath)) {
fs.delete(cuboidPath, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..31864f6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -55,6 +55,7 @@ import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -136,7 +137,7 @@ public class CubeMigrationCLI {
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
hbaseAdmin = new HBaseAdmin(conf);
- hdfsFS = FileSystem.get(new Configuration());
+ hdfsFS = HadoopUtil.getWorkingFileSystem();
operations = new ArrayList<Opt>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 9fe5a23..d1a74ad 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -50,6 +50,7 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -180,7 +181,7 @@ public class StorageCleanupJob extends AbstractApplication {
JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- FileSystem fs = FileSystem.get(conf);
+ FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
List<String> allHdfsPathsNeedToBeDeleted = new ArrayList<String>();
// GlobFilter filter = new
// GlobFilter(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index 40306c9..5269195 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -55,6 +55,7 @@ import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -134,7 +135,7 @@ public class CubeMigrationCLI {
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
hbaseAdmin = new HBaseAdmin(conf);
- hdfsFS = FileSystem.get(new Configuration());
+ hdfsFS = HadoopUtil.getWorkingFileSystem();
operations = new ArrayList<Opt>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa6cbe5b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index 05e0142..b3e2ec0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -185,7 +185,7 @@ public class StorageCleanupJob extends AbstractApplication {
JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- FileSystem fs = FileSystem.get(conf);
+ FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
List<String> allHdfsPathsNeedToBeDeleted = new ArrayList<String>();
// GlobFilter filter = new
// GlobFilter(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()
@@ -342,7 +342,7 @@ public class StorageCleanupJob extends AbstractApplication {
if (segmentId2JobId.containsKey(segmentId)) {
String path = JobBuilderSupport.getJobWorkingDir(engineConfig.getHdfsWorkingDirectory(), segmentId2JobId.get(segmentId)) + "/" + tableToDelete;
Path externalDataPath = new Path(path);
- FileSystem fs = FileSystem.get(externalDataPath.toUri(), HadoopUtil.getCurrentConfiguration());
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
if (fs.exists(externalDataPath)) {
fs.delete(externalDataPath, true);
logger.info("Hive table {}'s external path {} deleted", tableToDelete, path);