You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/10/13 00:16:14 UTC
[35/50] [abbrv] hive git commit: HIVE-15212 : merge branch into
master - clean up some pre-merge TODOs (Sergey Shelukhin)
http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
index 85d1324..0b7fe01 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
@@ -45,7 +45,6 @@ public class LoadTableDesc extends LoadDesc implements Serializable {
// TODO: the below seems like they should just be combined into partitionDesc
private org.apache.hadoop.hive.ql.plan.TableDesc table;
private Map<String, String> partitionSpec; // NOTE: this partitionSpec has to be ordered map
- private boolean commitMmWriteId = true;
public LoadTableDesc(final LoadTableDesc o) {
super(o.getSourcePath(), o.getWriteType());
@@ -65,8 +64,10 @@ public class LoadTableDesc extends LoadDesc implements Serializable {
final boolean replace,
final AcidUtils.Operation writeType, Long currentTransactionId) {
super(sourcePath, writeType);
- Utilities.LOG14535.info("creating part LTD from " + sourcePath + " to "
+ if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+ Utilities.FILE_OP_LOGGER.trace("creating part LTD from " + sourcePath + " to "
+ ((table.getProperties() == null) ? "null" : table.getTableName()));
+ }
init(table, partitionSpec, replace, currentTransactionId);
}
@@ -110,7 +111,9 @@ public class LoadTableDesc extends LoadDesc implements Serializable {
final AcidUtils.Operation writeType,
boolean isReplace, Long txnId) {
super(sourcePath, writeType);
- Utilities.LOG14535.info("creating LTD from " + sourcePath + " to " + table.getTableName()/*, new Exception()*/);
+ if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+ Utilities.FILE_OP_LOGGER.trace("creating LTD from " + sourcePath + " to " + table.getTableName());
+ }
this.dpCtx = dpCtx;
if (dpCtx != null && dpCtx.getPartSpec() != null && partitionSpec == null) {
init(table, dpCtx.getPartSpec(), isReplace, txnId);
@@ -211,12 +214,4 @@ public class LoadTableDesc extends LoadDesc implements Serializable {
public void setStmtId(int stmtId) {
this.stmtId = stmtId;
}
-
- public void setIntermediateInMmWrite(boolean b) {
- this.commitMmWriteId = !b;
- }
-
- public boolean isCommitMmWrite() {
- return commitMmWriteId;
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
index f6303ba..17d2f31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
@@ -80,8 +80,10 @@ public class MoveWork implements Serializable {
final LoadTableDesc loadTableWork, final LoadFileDesc loadFileWork,
boolean checkFileFormat, boolean srcLocal, LineageState lineageState) {
this(inputs, outputs, lineageState);
- Utilities.LOG14535.info("Creating MoveWork " + System.identityHashCode(this)
+ if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+ Utilities.FILE_OP_LOGGER.trace("Creating MoveWork " + System.identityHashCode(this)
+ " with " + loadTableWork + "; " + loadFileWork);
+ }
this.loadTableWork = loadTableWork;
this.loadFileWork = loadFileWork;
this.checkFileFormat = checkFileFormat;
http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java
index 03444c0..74e4387 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java
@@ -50,7 +50,9 @@ public class FSStatsAggregator implements StatsAggregator {
List<String> statsDirs = scc.getStatsTmpDirs();
assert statsDirs.size() == 1 : "Found multiple stats dirs: " + statsDirs;
Path statsDir = new Path(statsDirs.get(0));
- Utilities.LOG14535.info("About to read stats from : " + statsDir);
+ if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+ Utilities.FILE_OP_LOGGER.trace("About to read stats from : " + statsDir);
+ }
statsMap = new HashMap<String, Map<String,String>>();
try {
@@ -70,13 +72,12 @@ public class FSStatsAggregator implements StatsAggregator {
} finally {
SerializationUtilities.releaseKryo(kryo);
}
- Utilities.LOG14535.info("Read stats : " +statsMap);
statsList.add(statsMap);
in.close();
}
return true;
} catch (IOException e) {
- Utilities.LOG14535.error("Failed to read stats from filesystem ", e);
+ Utilities.FILE_OP_LOGGER.error("Failed to read stats from filesystem ", e);
return false;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
index a92465f..e8b878c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
@@ -77,7 +77,6 @@ public class FSStatsPublisher implements StatsPublisher {
@Override
public boolean publishStat(String partKV, Map<String, String> stats) {
- Utilities.LOG14535.info("Putting in map : " + partKV + "\t" + stats);
// we need to do new hashmap, since stats object is reused across calls.
Map<String,String> cpy = new HashMap<String, String>(stats);
Map<String,String> statMap = statsMap.get(partKV);
@@ -106,7 +105,9 @@ public class FSStatsPublisher implements StatsPublisher {
statsFile = new Path(statsDir, StatsSetupConst.STATS_FILE_PREFIX
+ conf.getInt("mapred.task.partition", 0));
}
- Utilities.LOG14535.info("About to create stats file for this task : " + statsFile);
+ if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+ Utilities.FILE_OP_LOGGER.trace("About to create stats file for this task : " + statsFile);
+ }
Output output = new Output(statsFile.getFileSystem(conf).create(statsFile,true));
LOG.debug("Created file : " + statsFile);
LOG.debug("Writing stats in it : " + statsMap);
@@ -119,7 +120,7 @@ public class FSStatsPublisher implements StatsPublisher {
output.close();
return true;
} catch (IOException e) {
- Utilities.LOG14535.error("Failed to persist stats on filesystem",e);
+ Utilities.FILE_OP_LOGGER.error("Failed to persist stats on filesystem",e);
return false;
}
}