You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/12/02 21:32:55 UTC
[2/3] hive git commit: HIVE-15312 : reduce logging in certain places
(Sergey Shelukhin, reviewed by Prasanth Jayachandran)
HIVE-15312 : reduce logging in certain places (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2f9728ed
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2f9728ed
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2f9728ed
Branch: refs/heads/master
Commit: 2f9728ed3507247270df99bd9c178c3f928cc463
Parents: 98a25f2
Author: Sergey Shelukhin <se...@apache.org>
Authored: Fri Dec 2 11:25:16 2016 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Fri Dec 2 13:32:27 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hive/common/util/Ref.java | 32 +++++++++++++++
.../llap/shufflehandler/ShuffleHandler.java | 2 +-
.../tezplugins/LlapTaskSchedulerService.java | 6 +--
.../apache/hadoop/hive/ql/exec/Utilities.java | 18 +++++----
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 32 ++++++++++++---
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 41 ++++++++++++++++----
.../apache/hadoop/hive/ql/io/orc/OrcSplit.java | 6 ++-
7 files changed, 112 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2f9728ed/common/src/java/org/apache/hive/common/util/Ref.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/common/util/Ref.java b/common/src/java/org/apache/hive/common/util/Ref.java
new file mode 100644
index 0000000..0f666dd
--- /dev/null
+++ b/common/src/java/org/apache/hive/common/util/Ref.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.common.util;
+
+/** Reference to T. */
+public final class Ref<T> {
+ public T value;
+
+ public Ref(T value) {
+ this.value = value;
+ }
+
+ public static <T> Ref<T> from(T t) {
+ return new Ref<T>(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/2f9728ed/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java b/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
index 9a3e221..02f7911 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
@@ -838,7 +838,7 @@ public class ShuffleHandler implements AttemptRegistrationListener {
response.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
response.setHeader(HttpHeaders.Values.KEEP_ALIVE, "timeout="
+ connectionKeepAliveTimeOut);
- LOG.info("Content Length in shuffle : " + contentLength);
+ LOG.debug("Content Length in shuffle : " + contentLength);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2f9728ed/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
index 158772b..7838bef 100644
--- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
@@ -1211,7 +1211,7 @@ public class LlapTaskSchedulerService extends TaskScheduler {
}
} else {
// No tasks qualify as preemptable
- LOG.info("No tasks qualify as killable to schedule tasks at priority {}", forPriority);
+ LOG.debug("No tasks qualify as killable to schedule tasks at priority {}", forPriority);
break;
}
}
@@ -1602,8 +1602,8 @@ public class LlapTaskSchedulerService extends TaskScheduler {
boolean canAcceptTask() {
boolean result = !hadCommFailure && !disabled
&&(numSchedulableTasks == -1 || ((numSchedulableTasks - numScheduledTasks) > 0));
- if (LOG.isInfoEnabled()) {
- LOG.info("Node[" + serviceInstance.getHost() + ":" + serviceInstance.getRpcPort() + ", " +
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Node[" + serviceInstance.getHost() + ":" + serviceInstance.getRpcPort() + ", " +
serviceInstance.getWorkerIdentity() + "]: " +
"canAcceptTask={}, numScheduledTasks={}, numSchedulableTasks={}, hadCommFailure={}, disabled={}",
result, numScheduledTasks, numSchedulableTasks, hadCommFailure, disabled);
http://git-wip-us.apache.org/repos/asf/hive/blob/2f9728ed/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 2b1d1ce..e8f50f2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -2974,6 +2974,8 @@ public final class Utilities {
// The alias may not have any path
Path path = null;
+ boolean hasLogged = false;
+ // Note: this copies the list because createDummyFileForEmptyPartition may modify the map.
for (Path file : new LinkedList<Path>(work.getPathToAliases().keySet())) {
List<String> aliases = work.getPathToAliases().get(file);
if (aliases.contains(alias)) {
@@ -2986,13 +2988,15 @@ public final class Utilities {
}
pathsProcessed.add(path);
-
- LOG.info("Adding input file " + path);
- if (!skipDummy
- && isEmptyPath(job, path, ctx)) {
- path = createDummyFileForEmptyPartition(path, job, work,
- hiveScratchDir);
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding input file " + path);
+ } else if (!hasLogged) {
+ hasLogged = true;
+ LOG.info("Adding " + work.getPathToAliases().size()
+ + " inputs; the first input is " + path);
+ }
+ if (!skipDummy && isEmptyPath(job, path, ctx)) {
+ path = createDummyFileForEmptyPartition(path, job, work, hiveScratchDir);
}
pathsToAdd.add(path);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2f9728ed/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index f1eba5d..da00bb3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.common.util.Ref;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -763,6 +764,15 @@ public class AcidUtils {
boolean useFileIds,
boolean ignoreEmptyFiles
) throws IOException {
+ return getAcidState(directory, conf, txnList, Ref.from(useFileIds), ignoreEmptyFiles);
+ }
+
+ public static Directory getAcidState(Path directory,
+ Configuration conf,
+ ValidTxnList txnList,
+ Ref<Boolean> useFileIds,
+ boolean ignoreEmptyFiles
+ ) throws IOException {
FileSystem fs = directory.getFileSystem(conf);
// The following 'deltas' includes all kinds of delta files including insert & delete deltas.
final List<ParsedDelta> deltas = new ArrayList<ParsedDelta>();
@@ -770,12 +780,18 @@ public class AcidUtils {
List<FileStatus> originalDirectories = new ArrayList<FileStatus>();
final List<FileStatus> obsolete = new ArrayList<FileStatus>();
List<HdfsFileStatusWithId> childrenWithId = null;
- if (useFileIds) {
+ Boolean val = useFileIds.value;
+ if (val == null || val) {
try {
childrenWithId = SHIMS.listLocatedHdfsStatus(fs, directory, hiddenFileFilter);
+ if (val == null) {
+ useFileIds.value = true;
+ }
} catch (Throwable t) {
LOG.error("Failed to get files with ID; using regular API: " + t.getMessage());
- useFileIds = false;
+ if (val == null && t instanceof UnsupportedOperationException) {
+ useFileIds.value = false;
+ }
}
}
TxnBase bestBase = new TxnBase();
@@ -995,15 +1011,21 @@ public class AcidUtils {
* @throws IOException
*/
private static void findOriginals(FileSystem fs, FileStatus stat,
- List<HdfsFileStatusWithId> original, boolean useFileIds) throws IOException {
+ List<HdfsFileStatusWithId> original, Ref<Boolean> useFileIds) throws IOException {
assert stat.isDir();
List<HdfsFileStatusWithId> childrenWithId = null;
- if (useFileIds) {
+ Boolean val = useFileIds.value;
+ if (val == null || val) {
try {
childrenWithId = SHIMS.listLocatedHdfsStatus(fs, stat.getPath(), hiddenFileFilter);
+ if (val == null) {
+ useFileIds.value = true;
+ }
} catch (Throwable t) {
LOG.error("Failed to get files with ID; using regular API: " + t.getMessage());
- useFileIds = false;
+ if (val == null && t instanceof UnsupportedOperationException) {
+ useFileIds.value = false;
+ }
}
}
if (childrenWithId != null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/2f9728ed/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 3fe93ac..361901e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -106,6 +106,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hive.common.util.Ref;
import org.apache.orc.ColumnStatistics;
import org.apache.orc.OrcProto;
import org.apache.orc.OrcUtils;
@@ -1015,11 +1016,16 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
private final Context context;
private final FileSystem fs;
private final Path dir;
- private final boolean useFileIds;
+ private final Ref<Boolean> useFileIds;
private final UserGroupInformation ugi;
FileGenerator(Context context, FileSystem fs, Path dir, boolean useFileIds,
UserGroupInformation ugi) {
+ this(context, fs, dir, Ref.from(useFileIds), ugi);
+ }
+
+ FileGenerator(Context context, FileSystem fs, Path dir, Ref<Boolean> useFileIds,
+ UserGroupInformation ugi) {
this.context = context;
this.fs = fs;
this.dir = dir;
@@ -1082,16 +1088,23 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
} else {
// This is a normal insert delta, which only has insert events and hence all the files
// in this delta directory can be considered as a base.
- if (useFileIds) {
+ Boolean val = useFileIds.value;
+ if (val == null || val) {
try {
List<HdfsFileStatusWithId> insertDeltaFiles =
SHIMS.listLocatedHdfsStatus(fs, parsedDelta.getPath(), AcidUtils.hiddenFileFilter);
for (HdfsFileStatusWithId fileId : insertDeltaFiles) {
baseFiles.add(new AcidBaseFileInfo(fileId, AcidUtils.AcidBaseFileType.INSERT_DELTA));
}
+ if (val == null) {
+ useFileIds.value = true; // The call succeeded, so presumably the API is there.
+ }
continue; // move on to process to the next parsedDelta.
} catch (Throwable t) {
LOG.error("Failed to get files with ID; using regular API: " + t.getMessage());
+ if (val == null && t instanceof UnsupportedOperationException) {
+ useFileIds.value = false;
+ }
}
}
// Fall back to regular API and create statuses without ID.
@@ -1112,12 +1125,21 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
}
private List<HdfsFileStatusWithId> findBaseFiles(
- Path base, boolean useFileIds) throws IOException {
- if (useFileIds) {
+ Path base, Ref<Boolean> useFileIds) throws IOException {
+ Boolean val = useFileIds.value;
+ if (val == null || val) {
try {
- return SHIMS.listLocatedHdfsStatus(fs, base, AcidUtils.hiddenFileFilter);
+ List<HdfsFileStatusWithId> result = SHIMS.listLocatedHdfsStatus(
+ fs, base, AcidUtils.hiddenFileFilter);
+ if (val == null) {
+ useFileIds.value = true; // The call succeeded, so presumably the API is there.
+ }
+ return result;
} catch (Throwable t) {
LOG.error("Failed to get files with ID; using regular API: " + t.getMessage());
+ if (val == null && t instanceof UnsupportedOperationException) {
+ useFileIds.value = false;
+ }
}
}
@@ -1542,8 +1564,13 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
if (LOG.isInfoEnabled()) {
LOG.info("ORC pushdown predicate: " + context.sarg);
}
- boolean useFileIds = HiveConf.getBoolVar(conf, ConfVars.HIVE_ORC_INCLUDE_FILE_ID_IN_SPLITS);
- boolean allowSyntheticFileIds = useFileIds && HiveConf.getBoolVar(
+ boolean useFileIdsConfig = HiveConf.getBoolVar(
+ conf, ConfVars.HIVE_ORC_INCLUDE_FILE_ID_IN_SPLITS);
+ // Sharing this state assumes splits will succeed or fail to get it together (same FS).
+ // We also start with null and only set it to true on the first call, so we would only do
+ // the global-disable thing on the first failure w/the API error, not any random failure.
+ Ref<Boolean> useFileIds = Ref.from(useFileIdsConfig ? null : false);
+ boolean allowSyntheticFileIds = useFileIdsConfig && HiveConf.getBoolVar(
conf, ConfVars.HIVE_ORC_ALLOW_SYNTHETIC_FILE_ID_IN_SPLITS);
List<OrcSplit> splits = Lists.newArrayList();
List<Future<AcidDirInfo>> pathFutures = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/hive/blob/2f9728ed/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
index 969e70e..d61b24b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
@@ -98,8 +98,10 @@ public class OrcSplit extends FileSplit implements ColumnarSplit, LlapAwareSplit
int additional = bos.size() - required;
out.write(bos.toByteArray());
- LOG.info("Writing additional {} bytes to OrcSplit as payload. Required {} bytes.", additional,
- required);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Writing additional {} bytes to OrcSplit as payload. Required {} bytes.",
+ additional, required);
+ }
}
private void writeAdditionalPayload(final DataOutputStream out) throws IOException {