You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ap...@apache.org on 2017/02/13 23:51:10 UTC
[2/4] phoenix git commit: PHOENIX-3661 Make phoenix tool select file
system dynamically (Yishan Yang)
PHOENIX-3661 Make phoenix tool select file system dynamically (Yishan Yang)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/222388b0
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/222388b0
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/222388b0
Branch: refs/heads/4.x-HBase-1.1
Commit: 222388b03415caad37d858d1cd91fe79be571787
Parents: beea861
Author: Andrew Purtell <ap...@apache.org>
Authored: Mon Feb 13 15:24:01 2017 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Mon Feb 13 15:25:30 2017 -0800
----------------------------------------------------------------------
.../apache/phoenix/mapreduce/AbstractBulkLoadTool.java | 2 +-
.../phoenix/mapreduce/MultiHfileOutputFormat.java | 2 +-
.../org/apache/phoenix/mapreduce/index/IndexTool.java | 12 ++++++++----
3 files changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/222388b0/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
index b32f9c6..f717647 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
@@ -331,7 +331,7 @@ public abstract class AbstractBulkLoadTool extends Configured implements Tool {
LOG.info("Loading HFiles from {}", outputPath);
completebulkload(conf,outputPath,tablesToBeLoaded);
LOG.info("Removing output directory {}", outputPath);
- if(!FileSystem.get(conf).delete(outputPath, true)) {
+ if(!outputPath.getFileSystem(conf).delete(outputPath, true)) {
LOG.error("Failed to delete the output directory {}", outputPath);
}
return 0;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/222388b0/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java
index 35a2bd8..da78fd5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java
@@ -455,8 +455,8 @@ public class MultiHfileOutputFormat extends FileOutputFormat<TableRowkeyPair, Ce
Configuration conf = job.getConfiguration();
// create the partitions file
- FileSystem fs = FileSystem.get(conf);
Path partitionsPath = new Path(conf.get("hadoop.tmp.dir"), "partitions_" + UUID.randomUUID());
+ FileSystem fs = partitionsPath.getFileSystem(conf);
fs.makeQualified(partitionsPath);
writePartitions(conf, partitionsPath, tablesStartKeys);
fs.deleteOnExit(partitionsPath);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/222388b0/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
index e594e0d..3606593 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
@@ -194,6 +194,7 @@ public class IndexTool extends Configured implements Tool {
Connection connection;
Configuration configuration;
private Path outputPath;
+ private FileSystem fs;
public JobFactory(Connection connection, Configuration configuration, Path outputPath) {
this.connection = connection;
@@ -357,8 +358,9 @@ public class IndexTool extends Configured implements Tool {
final List<ColumnInfo> columnMetadataList =
PhoenixRuntime.generateColumnInfo(connection, qIndexTable, indexColumns);
ColumnInfoToStringEncoderDecoder.encode(configuration, columnMetadataList);
- FileSystem.get(configuration).delete(outputPath, true);
-
+ fs = outputPath.getFileSystem(configuration);
+ fs.delete(outputPath, true);
+
final String jobName = String.format(INDEX_JOB_NAME_TEMPLATE, pdataTable.getName().toString(), indexTable);
final Job job = Job.getInstance(configuration, jobName);
job.setJarByClass(IndexTool.class);
@@ -475,10 +477,12 @@ public class IndexTool extends Configured implements Tool {
PTable pdataTable = PhoenixRuntime.getTableNoCache(connection, qDataTable);
Path outputPath = null;
+ FileSystem fs = null;
if (basePath != null) {
outputPath = CsvBulkImportUtil.getOutputPath(new Path(basePath), pindexTable == null
? pdataTable.getPhysicalName().getString() : pindexTable.getPhysicalName().getString());
- FileSystem.get(configuration).delete(outputPath, true);
+ fs = outputPath.getFileSystem(configuration);
+ fs.delete(outputPath, true);
}
Job job = new JobFactory(connection, configuration, outputPath).getJob(schemaName, indexTable, dataTable,
@@ -502,7 +506,7 @@ public class IndexTool extends Configured implements Tool {
htable.close();
// Without direct API, we need to update the index state to ACTIVE from client.
IndexToolUtil.updateIndexState(connection, qDataTable, indexTable, PIndexState.ACTIVE);
- FileSystem.get(configuration).delete(outputPath, true);
+ fs.delete(outputPath, true);
}
return 0;
} else {