You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/04/16 00:04:07 UTC
svn commit: r1673969 [19/19] - in /hive/branches/llap: ./
beeline/src/java/org/apache/hive/beeline/ bin/
cli/src/java/org/apache/hadoop/hive/cli/
cli/src/test/org/apache/hadoop/hive/cli/ common/
common/src/java/org/apache/hadoop/hive/common/jsonexplain...
Modified: hive/branches/llap/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java (original)
+++ hive/branches/llap/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java Wed Apr 15 22:04:00 2015
@@ -31,7 +31,9 @@ public enum TTypeId implements org.apach
NULL_TYPE(16),
DATE_TYPE(17),
VARCHAR_TYPE(18),
- CHAR_TYPE(19);
+ CHAR_TYPE(19),
+ INTERVAL_YEAR_MONTH_TYPE(20),
+ INTERVAL_DAY_TIME_TYPE(21);
private final int value;
@@ -92,6 +94,10 @@ public enum TTypeId implements org.apach
return VARCHAR_TYPE;
case 19:
return CHAR_TYPE;
+ case 20:
+ return INTERVAL_YEAR_MONTH_TYPE;
+ case 21:
+ return INTERVAL_DAY_TIME_TYPE;
default:
return null;
}
Modified: hive/branches/llap/service/src/gen/thrift/gen-py/TCLIService/constants.py
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/gen/thrift/gen-py/TCLIService/constants.py?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/service/src/gen/thrift/gen-py/TCLIService/constants.py (original)
+++ hive/branches/llap/service/src/gen/thrift/gen-py/TCLIService/constants.py Wed Apr 15 22:04:00 2015
@@ -25,6 +25,8 @@ PRIMITIVE_TYPES = set([
17,
18,
19,
+ 20,
+ 21,
])
COMPLEX_TYPES = set([
10,
@@ -57,6 +59,8 @@ TYPE_NAMES = {
17 : "DATE",
18 : "VARCHAR",
19 : "CHAR",
+ 20 : "INTERVAL_YEAR_MONTH",
+ 21 : "INTERVAL_DAY_TIME",
}
CHARACTER_MAXIMUM_LENGTH = "characterMaximumLength"
PRECISION = "precision"
Modified: hive/branches/llap/service/src/gen/thrift/gen-py/TCLIService/ttypes.py
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/gen/thrift/gen-py/TCLIService/ttypes.py?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/service/src/gen/thrift/gen-py/TCLIService/ttypes.py (original)
+++ hive/branches/llap/service/src/gen/thrift/gen-py/TCLIService/ttypes.py Wed Apr 15 22:04:00 2015
@@ -24,6 +24,7 @@ class TProtocolVersion:
HIVE_CLI_SERVICE_PROTOCOL_V5 = 4
HIVE_CLI_SERVICE_PROTOCOL_V6 = 5
HIVE_CLI_SERVICE_PROTOCOL_V7 = 6
+ HIVE_CLI_SERVICE_PROTOCOL_V8 = 7
_VALUES_TO_NAMES = {
0: "HIVE_CLI_SERVICE_PROTOCOL_V1",
@@ -33,6 +34,7 @@ class TProtocolVersion:
4: "HIVE_CLI_SERVICE_PROTOCOL_V5",
5: "HIVE_CLI_SERVICE_PROTOCOL_V6",
6: "HIVE_CLI_SERVICE_PROTOCOL_V7",
+ 7: "HIVE_CLI_SERVICE_PROTOCOL_V8",
}
_NAMES_TO_VALUES = {
@@ -43,6 +45,7 @@ class TProtocolVersion:
"HIVE_CLI_SERVICE_PROTOCOL_V5": 4,
"HIVE_CLI_SERVICE_PROTOCOL_V6": 5,
"HIVE_CLI_SERVICE_PROTOCOL_V7": 6,
+ "HIVE_CLI_SERVICE_PROTOCOL_V8": 7,
}
class TTypeId:
@@ -66,6 +69,8 @@ class TTypeId:
DATE_TYPE = 17
VARCHAR_TYPE = 18
CHAR_TYPE = 19
+ INTERVAL_YEAR_MONTH_TYPE = 20
+ INTERVAL_DAY_TIME_TYPE = 21
_VALUES_TO_NAMES = {
0: "BOOLEAN_TYPE",
@@ -88,6 +93,8 @@ class TTypeId:
17: "DATE_TYPE",
18: "VARCHAR_TYPE",
19: "CHAR_TYPE",
+ 20: "INTERVAL_YEAR_MONTH_TYPE",
+ 21: "INTERVAL_DAY_TIME_TYPE",
}
_NAMES_TO_VALUES = {
@@ -111,6 +118,8 @@ class TTypeId:
"DATE_TYPE": 17,
"VARCHAR_TYPE": 18,
"CHAR_TYPE": 19,
+ "INTERVAL_YEAR_MONTH_TYPE": 20,
+ "INTERVAL_DAY_TIME_TYPE": 21,
}
class TStatusCode:
@@ -3245,7 +3254,7 @@ class TOpenSessionReq:
thrift_spec = (
None, # 0
- (1, TType.I32, 'client_protocol', None, 6, ), # 1
+ (1, TType.I32, 'client_protocol', None, 7, ), # 1
(2, TType.STRING, 'username', None, None, ), # 2
(3, TType.STRING, 'password', None, None, ), # 3
(4, TType.MAP, 'configuration', (TType.STRING,None,TType.STRING,None), None, ), # 4
@@ -3354,7 +3363,7 @@ class TOpenSessionResp:
thrift_spec = (
None, # 0
(1, TType.STRUCT, 'status', (TStatus, TStatus.thrift_spec), None, ), # 1
- (2, TType.I32, 'serverProtocolVersion', None, 6, ), # 2
+ (2, TType.I32, 'serverProtocolVersion', None, 7, ), # 2
(3, TType.STRUCT, 'sessionHandle', (TSessionHandle, TSessionHandle.thrift_spec), None, ), # 3
(4, TType.MAP, 'configuration', (TType.STRING,None,TType.STRING,None), None, ), # 4
)
Modified: hive/branches/llap/service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb (original)
+++ hive/branches/llap/service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb Wed Apr 15 22:04:00 2015
@@ -23,6 +23,8 @@ PRIMITIVE_TYPES = Set.new([
17,
18,
19,
+ 20,
+ 21,
])
COMPLEX_TYPES = Set.new([
@@ -58,6 +60,8 @@ TYPE_NAMES = {
17 => %q"DATE",
18 => %q"VARCHAR",
19 => %q"CHAR",
+ 20 => %q"INTERVAL_YEAR_MONTH",
+ 21 => %q"INTERVAL_DAY_TIME",
}
CHARACTER_MAXIMUM_LENGTH = %q"characterMaximumLength"
Modified: hive/branches/llap/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb (original)
+++ hive/branches/llap/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb Wed Apr 15 22:04:00 2015
@@ -14,8 +14,9 @@ module TProtocolVersion
HIVE_CLI_SERVICE_PROTOCOL_V5 = 4
HIVE_CLI_SERVICE_PROTOCOL_V6 = 5
HIVE_CLI_SERVICE_PROTOCOL_V7 = 6
- VALUE_MAP = {0 => "HIVE_CLI_SERVICE_PROTOCOL_V1", 1 => "HIVE_CLI_SERVICE_PROTOCOL_V2", 2 => "HIVE_CLI_SERVICE_PROTOCOL_V3", 3 => "HIVE_CLI_SERVICE_PROTOCOL_V4", 4 => "HIVE_CLI_SERVICE_PROTOCOL_V5", 5 => "HIVE_CLI_SERVICE_PROTOCOL_V6", 6 => "HIVE_CLI_SERVICE_PROTOCOL_V7"}
- VALID_VALUES = Set.new([HIVE_CLI_SERVICE_PROTOCOL_V1, HIVE_CLI_SERVICE_PROTOCOL_V2, HIVE_CLI_SERVICE_PROTOCOL_V3, HIVE_CLI_SERVICE_PROTOCOL_V4, HIVE_CLI_SERVICE_PROTOCOL_V5, HIVE_CLI_SERVICE_PROTOCOL_V6, HIVE_CLI_SERVICE_PROTOCOL_V7]).freeze
+ HIVE_CLI_SERVICE_PROTOCOL_V8 = 7
+ VALUE_MAP = {0 => "HIVE_CLI_SERVICE_PROTOCOL_V1", 1 => "HIVE_CLI_SERVICE_PROTOCOL_V2", 2 => "HIVE_CLI_SERVICE_PROTOCOL_V3", 3 => "HIVE_CLI_SERVICE_PROTOCOL_V4", 4 => "HIVE_CLI_SERVICE_PROTOCOL_V5", 5 => "HIVE_CLI_SERVICE_PROTOCOL_V6", 6 => "HIVE_CLI_SERVICE_PROTOCOL_V7", 7 => "HIVE_CLI_SERVICE_PROTOCOL_V8"}
+ VALID_VALUES = Set.new([HIVE_CLI_SERVICE_PROTOCOL_V1, HIVE_CLI_SERVICE_PROTOCOL_V2, HIVE_CLI_SERVICE_PROTOCOL_V3, HIVE_CLI_SERVICE_PROTOCOL_V4, HIVE_CLI_SERVICE_PROTOCOL_V5, HIVE_CLI_SERVICE_PROTOCOL_V6, HIVE_CLI_SERVICE_PROTOCOL_V7, HIVE_CLI_SERVICE_PROTOCOL_V8]).freeze
end
module TTypeId
@@ -39,8 +40,10 @@ module TTypeId
DATE_TYPE = 17
VARCHAR_TYPE = 18
CHAR_TYPE = 19
- VALUE_MAP = {0 => "BOOLEAN_TYPE", 1 => "TINYINT_TYPE", 2 => "SMALLINT_TYPE", 3 => "INT_TYPE", 4 => "BIGINT_TYPE", 5 => "FLOAT_TYPE", 6 => "DOUBLE_TYPE", 7 => "STRING_TYPE", 8 => "TIMESTAMP_TYPE", 9 => "BINARY_TYPE", 10 => "ARRAY_TYPE", 11 => "MAP_TYPE", 12 => "STRUCT_TYPE", 13 => "UNION_TYPE", 14 => "USER_DEFINED_TYPE", 15 => "DECIMAL_TYPE", 16 => "NULL_TYPE", 17 => "DATE_TYPE", 18 => "VARCHAR_TYPE", 19 => "CHAR_TYPE"}
- VALID_VALUES = Set.new([BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, TIMESTAMP_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, UNION_TYPE, USER_DEFINED_TYPE, DECIMAL_TYPE, NULL_TYPE, DATE_TYPE, VARCHAR_TYPE, CHAR_TYPE]).freeze
+ INTERVAL_YEAR_MONTH_TYPE = 20
+ INTERVAL_DAY_TIME_TYPE = 21
+ VALUE_MAP = {0 => "BOOLEAN_TYPE", 1 => "TINYINT_TYPE", 2 => "SMALLINT_TYPE", 3 => "INT_TYPE", 4 => "BIGINT_TYPE", 5 => "FLOAT_TYPE", 6 => "DOUBLE_TYPE", 7 => "STRING_TYPE", 8 => "TIMESTAMP_TYPE", 9 => "BINARY_TYPE", 10 => "ARRAY_TYPE", 11 => "MAP_TYPE", 12 => "STRUCT_TYPE", 13 => "UNION_TYPE", 14 => "USER_DEFINED_TYPE", 15 => "DECIMAL_TYPE", 16 => "NULL_TYPE", 17 => "DATE_TYPE", 18 => "VARCHAR_TYPE", 19 => "CHAR_TYPE", 20 => "INTERVAL_YEAR_MONTH_TYPE", 21 => "INTERVAL_DAY_TIME_TYPE"}
+ VALID_VALUES = Set.new([BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, TIMESTAMP_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, UNION_TYPE, USER_DEFINED_TYPE, DECIMAL_TYPE, NULL_TYPE, DATE_TYPE, VARCHAR_TYPE, CHAR_TYPE, INTERVAL_YEAR_MONTH_TYPE, INTERVAL_DAY_TIME_TYPE]).freeze
end
module TStatusCode
@@ -947,7 +950,7 @@ class TOpenSessionReq
CONFIGURATION = 4
FIELDS = {
- CLIENT_PROTOCOL => {:type => ::Thrift::Types::I32, :name => 'client_protocol', :default => 6, :enum_class => ::TProtocolVersion},
+ CLIENT_PROTOCOL => {:type => ::Thrift::Types::I32, :name => 'client_protocol', :default => 7, :enum_class => ::TProtocolVersion},
USERNAME => {:type => ::Thrift::Types::STRING, :name => 'username', :optional => true},
PASSWORD => {:type => ::Thrift::Types::STRING, :name => 'password', :optional => true},
CONFIGURATION => {:type => ::Thrift::Types::MAP, :name => 'configuration', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}, :optional => true}
@@ -974,7 +977,7 @@ class TOpenSessionResp
FIELDS = {
STATUS => {:type => ::Thrift::Types::STRUCT, :name => 'status', :class => ::TStatus},
- SERVERPROTOCOLVERSION => {:type => ::Thrift::Types::I32, :name => 'serverProtocolVersion', :default => 6, :enum_class => ::TProtocolVersion},
+ SERVERPROTOCOLVERSION => {:type => ::Thrift::Types::I32, :name => 'serverProtocolVersion', :default => 7, :enum_class => ::TProtocolVersion},
SESSIONHANDLE => {:type => ::Thrift::Types::STRUCT, :name => 'sessionHandle', :class => ::TSessionHandle, :optional => true},
CONFIGURATION => {:type => ::Thrift::Types::MAP, :name => 'configuration', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}, :optional => true}
}
Modified: hive/branches/llap/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java (original)
+++ hive/branches/llap/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java Wed Apr 15 22:04:00 2015
@@ -18,6 +18,9 @@
package org.apache.hive.service.cli;
+import org.apache.log4j.Layout;
+import org.apache.log4j.PatternLayout;
+
/**
* CLIServiceUtils.
*
@@ -26,6 +29,10 @@ public class CLIServiceUtils {
private static final char SEARCH_STRING_ESCAPE = '\\';
+ public static final Layout verboseLayout = new PatternLayout(
+ "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n");
+ public static final Layout nonVerboseLayout = new PatternLayout(
+ "%-5p : %m%n");
/**
* Convert a SQL search pattern into an equivalent Java Regex.
Modified: hive/branches/llap/service/src/java/org/apache/hive/service/cli/ColumnValue.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/java/org/apache/hive/service/cli/ColumnValue.java?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/service/src/java/org/apache/hive/service/cli/ColumnValue.java (original)
+++ hive/branches/llap/service/src/java/org/apache/hive/service/cli/ColumnValue.java Wed Apr 15 22:04:00 2015
@@ -24,6 +24,8 @@ import java.sql.Timestamp;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hive.service.cli.thrift.TBoolValue;
import org.apache.hive.service.cli.thrift.TByteValue;
@@ -144,6 +146,22 @@ public class ColumnValue {
return TColumnValue.stringVal(tStrValue);
}
+ private static TColumnValue stringValue(HiveIntervalYearMonth value) {
+ TStringValue tStrValue = new TStringValue();
+ if (value != null) {
+ tStrValue.setValue(value.toString());
+ }
+ return TColumnValue.stringVal(tStrValue);
+ }
+
+ private static TColumnValue stringValue(HiveIntervalDayTime value) {
+ TStringValue tStrValue = new TStringValue();
+ if (value != null) {
+ tStrValue.setValue(value.toString());
+ }
+ return TColumnValue.stringVal(tStrValue);
+ }
+
public static TColumnValue toTColumnValue(Type type, Object value) {
switch (type) {
case BOOLEAN_TYPE:
@@ -170,6 +188,10 @@ public class ColumnValue {
return dateValue((Date)value);
case TIMESTAMP_TYPE:
return timestampValue((Timestamp)value);
+ case INTERVAL_YEAR_MONTH_TYPE:
+ return stringValue((HiveIntervalYearMonth) value);
+ case INTERVAL_DAY_TIME_TYPE:
+ return stringValue((HiveIntervalDayTime) value);
case DECIMAL_TYPE:
return stringValue(((HiveDecimal)value));
case BINARY_TYPE:
Modified: hive/branches/llap/service/src/java/org/apache/hive/service/cli/Type.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/java/org/apache/hive/service/cli/Type.java?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/service/src/java/org/apache/hive/service/cli/Type.java (original)
+++ hive/branches/llap/service/src/java/org/apache/hive/service/cli/Type.java Wed Apr 15 22:04:00 2015
@@ -69,6 +69,12 @@ public enum Type {
TIMESTAMP_TYPE("TIMESTAMP",
java.sql.Types.TIMESTAMP,
TTypeId.TIMESTAMP_TYPE),
+ INTERVAL_YEAR_MONTH_TYPE("INTERVAL_YEAR_MONTH",
+ java.sql.Types.OTHER,
+ TTypeId.INTERVAL_YEAR_MONTH_TYPE),
+ INTERVAL_DAY_TIME_TYPE("INTERVAL_DAY_TIME",
+ java.sql.Types.OTHER,
+ TTypeId.INTERVAL_DAY_TIME_TYPE),
BINARY_TYPE("BINARY",
java.sql.Types.BINARY,
TTypeId.BINARY_TYPE),
Modified: hive/branches/llap/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java (original)
+++ hive/branches/llap/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java Wed Apr 15 22:04:00 2015
@@ -18,10 +18,16 @@
package org.apache.hive.service.cli.operation;
import java.io.CharArrayWriter;
+import java.util.Enumeration;
import java.util.regex.Pattern;
import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.session.OperationLog;
+import org.apache.hadoop.hive.ql.session.OperationLog.LoggingLevel;
+import org.apache.hive.service.cli.CLIServiceUtils;
+import org.apache.log4j.Appender;
+import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Layout;
import org.apache.log4j.Logger;
import org.apache.log4j.WriterAppender;
@@ -36,6 +42,8 @@ import com.google.common.base.Joiner;
public class LogDivertAppender extends WriterAppender {
private static final Logger LOG = Logger.getLogger(LogDivertAppender.class.getName());
private final OperationManager operationManager;
+ private boolean isVerbose;
+ private Layout verboseLayout;
/**
* A log filter that filters messages coming from the logger with the given names.
@@ -45,18 +53,71 @@ public class LogDivertAppender extends W
* White list filter is used for less verbose log collection
*/
private static class NameFilter extends Filter {
- private final Pattern namePattern;
- private final boolean excludeMatches;
+ private Pattern namePattern;
+ private LoggingLevel loggingMode;
+ private OperationManager operationManager;
+
+ /* Patterns that are excluded in verbose logging level.
+ * Filter out messages coming from log processing classes, or we'll run an infinite loop.
+ */
+ private static final Pattern verboseExcludeNamePattern = Pattern.compile(Joiner.on("|").
+ join(new String[] {LOG.getName(), OperationLog.class.getName(),
+ OperationManager.class.getName()}));
+
+ /* Patterns that are included in execution logging level.
+ * In execution mode, show only select logger messages.
+ */
+ private static final Pattern executionIncludeNamePattern = Pattern.compile(Joiner.on("|").
+ join(new String[] {"org.apache.hadoop.mapreduce.JobSubmitter",
+ "org.apache.hadoop.mapreduce.Job", "SessionState", Task.class.getName(),
+ "org.apache.hadoop.hive.ql.exec.spark.status.SparkJobMonitor"}));
+
+ /* Patterns that are included in performance logging level.
+ * In performance mode, show execution and performance logger messages.
+ */
+ private static final Pattern performanceIncludeNamePattern = Pattern.compile(
+ executionIncludeNamePattern.pattern() + "|" + PerfLogger.class.getName());
+
+ private void setCurrentNamePattern(OperationLog.LoggingLevel mode) {
+ if (mode == OperationLog.LoggingLevel.VERBOSE) {
+ this.namePattern = verboseExcludeNamePattern;
+ } else if (mode == OperationLog.LoggingLevel.EXECUTION) {
+ this.namePattern = executionIncludeNamePattern;
+ } else if (mode == OperationLog.LoggingLevel.PERFORMANCE) {
+ this.namePattern = performanceIncludeNamePattern;
+ }
+ }
- public NameFilter(boolean isExclusionFilter, String [] loggerNames) {
- this.excludeMatches = isExclusionFilter;
- String matchRegex = Joiner.on("|").join(loggerNames);
- this.namePattern = Pattern.compile(matchRegex);
+ public NameFilter(
+ OperationLog.LoggingLevel loggingMode, OperationManager op) {
+ this.operationManager = op;
+ this.loggingMode = loggingMode;
+ setCurrentNamePattern(loggingMode);
}
@Override
public int decide(LoggingEvent ev) {
+ OperationLog log = operationManager.getOperationLogByThread();
+ boolean excludeMatches = (loggingMode == OperationLog.LoggingLevel.VERBOSE);
+
+ if (log == null) {
+ return Filter.DENY;
+ }
+
+ OperationLog.LoggingLevel currentLoggingMode = log.getOpLoggingLevel();
+ // If logging is disabled, deny everything.
+ if (currentLoggingMode == OperationLog.LoggingLevel.NONE) {
+ return Filter.DENY;
+ }
+ // Look at the current session's setting
+ // and set the pattern and excludeMatches accordingly.
+ if (currentLoggingMode != loggingMode) {
+ loggingMode = currentLoggingMode;
+ setCurrentNamePattern(loggingMode);
+ }
+
boolean isMatch = namePattern.matcher(ev.getLoggerName()).matches();
+
if (excludeMatches == isMatch) {
// Deny if this is black-list filter (excludeMatches = true) and it
// matched
@@ -70,25 +131,61 @@ public class LogDivertAppender extends W
/** This is where the log message will go to */
private final CharArrayWriter writer = new CharArrayWriter();
- public LogDivertAppender(Layout layout, OperationManager operationManager, boolean isVerbose) {
- setLayout(layout);
+ private void setLayout (boolean isVerbose, Layout lo) {
+ if (isVerbose) {
+ if (lo == null) {
+ lo = CLIServiceUtils.verboseLayout;
+ LOG.info("Cannot find a Layout from a ConsoleAppender. Using default Layout pattern.");
+ }
+ } else {
+ lo = CLIServiceUtils.nonVerboseLayout;
+ }
+ setLayout(lo);
+ }
+
+ private void initLayout(boolean isVerbose) {
+ // There should be a ConsoleAppender. Copy its Layout.
+ Logger root = Logger.getRootLogger();
+ Layout layout = null;
+
+ Enumeration<?> appenders = root.getAllAppenders();
+ while (appenders.hasMoreElements()) {
+ Appender ap = (Appender) appenders.nextElement();
+ if (ap.getClass().equals(ConsoleAppender.class)) {
+ layout = ap.getLayout();
+ break;
+ }
+ }
+ setLayout(isVerbose, layout);
+ }
+
+ public LogDivertAppender(OperationManager operationManager,
+ OperationLog.LoggingLevel loggingMode) {
+ isVerbose = (loggingMode == OperationLog.LoggingLevel.VERBOSE);
+ initLayout(isVerbose);
setWriter(writer);
setName("LogDivertAppender");
this.operationManager = operationManager;
+ this.verboseLayout = isVerbose ? layout : CLIServiceUtils.verboseLayout;
+ addFilter(new NameFilter(loggingMode, operationManager));
+ }
- if (isVerbose) {
- // Filter out messages coming from log processing classes, or we'll run an
- // infinite loop.
- String[] exclLoggerNames = { LOG.getName(), OperationLog.class.getName(),
- OperationManager.class.getName() };
- addFilter(new NameFilter(true, exclLoggerNames));
- } else {
- // in non verbose mode, show only select logger messages
- String[] inclLoggerNames = { "org.apache.hadoop.mapreduce.JobSubmitter",
- "org.apache.hadoop.mapreduce.Job", "SessionState", Task.class.getName(),
- "org.apache.hadoop.hive.ql.exec.spark.status.SparkJobMonitor"};
- addFilter(new NameFilter(false, inclLoggerNames));
+ @Override
+ public void doAppend(LoggingEvent event) {
+ OperationLog log = operationManager.getOperationLogByThread();
+
+ // Set current layout depending on the verbose/non-verbose mode.
+ if (log != null) {
+ boolean isCurrModeVerbose = (log.getOpLoggingLevel() == OperationLog.LoggingLevel.VERBOSE);
+
+ // If there is a logging level change from verbose->non-verbose or vice-versa since
+ // the last subAppend call, change the layout to preserve consistency.
+ if (isCurrModeVerbose != isVerbose) {
+ isVerbose = isCurrModeVerbose;
+ setLayout(isVerbose, verboseLayout);
+ }
}
+ super.doAppend(event);
}
/**
Modified: hive/branches/llap/service/src/java/org/apache/hive/service/cli/operation/Operation.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/java/org/apache/hive/service/cli/operation/Operation.java?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/service/src/java/org/apache/hive/service/cli/operation/Operation.java (original)
+++ hive/branches/llap/service/src/java/org/apache/hive/service/cli/operation/Operation.java Wed Apr 15 22:04:00 2015
@@ -210,7 +210,7 @@ public abstract class Operation {
// create OperationLog object with above log file
try {
- operationLog = new OperationLog(opHandle.toString(), operationLogFile);
+ operationLog = new OperationLog(opHandle.toString(), operationLogFile, parentSession.getHiveConf());
} catch (FileNotFoundException e) {
LOG.warn("Unable to instantiate OperationLog object for operation: " +
opHandle, e);
Modified: hive/branches/llap/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java (original)
+++ hive/branches/llap/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java Wed Apr 15 22:04:00 2015
@@ -20,7 +20,6 @@ package org.apache.hive.service.cli.oper
import java.sql.SQLException;
import java.util.ArrayList;
-import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -42,10 +41,7 @@ import org.apache.hive.service.cli.RowSe
import org.apache.hive.service.cli.TableSchema;
import org.apache.hive.service.cli.session.HiveSession;
import org.apache.log4j.Appender;
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.Layout;
import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
/**
* OperationManager.
@@ -54,7 +50,6 @@ import org.apache.log4j.PatternLayout;
public class OperationManager extends AbstractService {
private final Log LOG = LogFactory.getLog(OperationManager.class.getName());
- private HiveConf hiveConf;
private final Map<OperationHandle, Operation> handleToOperation =
new HashMap<OperationHandle, Operation>();
@@ -64,10 +59,9 @@ public class OperationManager extends Ab
@Override
public synchronized void init(HiveConf hiveConf) {
- this.hiveConf = hiveConf;
if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED)) {
- boolean isVerbose = hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_VERBOSE);
- initOperationLogCapture(isVerbose);
+ initOperationLogCapture(hiveConf.getVar(
+ HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL));
} else {
LOG.debug("Operation level logging is turned off");
}
@@ -86,34 +80,10 @@ public class OperationManager extends Ab
super.stop();
}
- private void initOperationLogCapture(boolean isVerbose) {
- // There should be a ConsoleAppender. Copy its Layout.
- Logger root = Logger.getRootLogger();
- Layout layout = null;
-
- Enumeration<?> appenders = root.getAllAppenders();
- while (appenders.hasMoreElements()) {
- Appender ap = (Appender) appenders.nextElement();
- if (ap.getClass().equals(ConsoleAppender.class)) {
- layout = ap.getLayout();
- break;
- }
- }
-
- final String VERBOSE_PATTERN = "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n";
- final String NONVERBOSE_PATTERN = "%-5p : %m%n";
-
- if (isVerbose) {
- if (layout == null) {
- layout = new PatternLayout(VERBOSE_PATTERN);
- LOG.info("Cannot find a Layout from a ConsoleAppender. Using default Layout pattern.");
- }
- } else {
- layout = new PatternLayout(NONVERBOSE_PATTERN);
- }
+ private void initOperationLogCapture(String loggingMode) {
// Register another Appender (with the same layout) that talks to us.
- Appender ap = new LogDivertAppender(layout, this, isVerbose);
- root.addAppender(ap);
+ Appender ap = new LogDivertAppender(this, OperationLog.getLoggingLevel(loggingMode));
+ Logger.getRootLogger().addAppender(ap);
}
public ExecuteStatementOperation newExecuteStatementOperation(HiveSession parentSession,
Modified: hive/branches/llap/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/branches/llap/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Wed Apr 15 22:04:00 2015
@@ -156,6 +156,11 @@ public class Hadoop20SShims extends Hado
}
@Override
+ public void startPauseMonitor(Configuration conf) {
+ /* no supported */
+ }
+
+ @Override
public boolean isLocalMode(Configuration conf) {
return "local".equals(getJobLauncherRpcAddress(conf));
}
Modified: hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/branches/llap/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Wed Apr 15 22:04:00 2015
@@ -213,6 +213,19 @@ public class Hadoop23Shims extends Hadoo
}
@Override
+ public void startPauseMonitor(Configuration conf) {
+ try {
+ Class.forName("org.apache.hadoop.util.JvmPauseMonitor");
+ org.apache.hadoop.util.JvmPauseMonitor pauseMonitor = new org.apache.hadoop.util
+ .JvmPauseMonitor(conf);
+ pauseMonitor.start();
+ } catch (Throwable t) {
+ LOG.warn("Could not initiate the JvmPauseMonitor thread." + " GCs and Pauses may not be " +
+ "warned upon.", t);
+ }
+ }
+
+ @Override
public boolean isLocalMode(Configuration conf) {
return "local".equals(conf.get("mapreduce.framework.name"));
}
Modified: hive/branches/llap/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/branches/llap/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java Wed Apr 15 22:04:00 2015
@@ -144,6 +144,8 @@ public interface HadoopShims {
public JobContext newJobContext(Job job);
+ public void startPauseMonitor(Configuration conf);
+
/**
* Check wether MR is configured to run in local-mode
* @param conf
Modified: hive/branches/llap/testutils/metastore/execute-test-on-lxc.sh
URL: http://svn.apache.org/viewvc/hive/branches/llap/testutils/metastore/execute-test-on-lxc.sh?rev=1673969&r1=1673968&r2=1673969&view=diff
==============================================================================
--- hive/branches/llap/testutils/metastore/execute-test-on-lxc.sh (original)
+++ hive/branches/llap/testutils/metastore/execute-test-on-lxc.sh Wed Apr 15 22:04:00 2015
@@ -70,9 +70,11 @@ lxc_exists() {
}
lxc_create() {
- lxc-create -n $1 -t download -- --dist "ubuntu" --release "trusty" --arch "amd64"
+ lxc-create -n $1 -t download -- --dist "ubuntu" --release "trusty" --arch "amd64" || return 1
+ lxc_start $1 || return 1
lxc-attach -n $1 -- apt-get update
- lxc-attach -n $1 -- apt-get install -y openssh-server subversion
+ lxc-attach -n $1 -- apt-get install -y openssh-server subversion patch
+
printf "root\nroot" | sudo lxc-attach -n $1 -- passwd
lxc-attach -n $1 -- sed -i /etc/ssh/sshd_config 's/^PermitRootLogin without-password/PermitRootLogin yes/'
lxc-attach -n $1 -- service ssh restart
@@ -83,20 +85,29 @@ lxc_running() {
}
lxc_start() {
- lxc-start -n $1 --daemon
- lxc-wait -n mysql -s RUNNING
+ lxc-start -n $1 --daemon || return 1
+ lxc-wait -n $1 -s RUNNING || return 1
sleep 10 # wait a little longer
}
+lxc_stop() {
+ lxc-stop -n $1
+}
+
lxc_prepare() {
echo "Downloading hive source code from SVN, branch='$BRANCH' ..."
- lxc-attach -n $1 -- rm -rf /tmp/hive
- lxc-attach -n $1 -- mkdir /tmp/hive
- lxc-attach -n $1 -- svn co http://svn.apache.org/repos/asf/hive/$BRANCH /tmp/hive >/dev/null
+ tmpfile=$(mktemp)
+ cat>$tmpfile<<EOF
+rm -rf hive
+mkdir hive
+svn co http://svn.apache.org/repos/asf/hive/$BRANCH hive >/dev/null
+cd hive
+wget $PATCH_URL -O hms.patch
+bash -x testutils/ptest2/src/main/resources/smart-apply-patch.sh hms.patch
+EOF
- lxc-attach -n $1 -- wget $PATCH_URL -O /tmp/hive/hms.patch
- lxc-attach -n $1 -- patch -s -N -d /tmp/hive -p1 -i /tmp/hive/hms.patch
+ lxc-attach -n $1 -- bash -x -e < $tmpfile
}
lxc_print_metastore_log() {
@@ -104,7 +115,7 @@ lxc_print_metastore_log() {
}
run_tests() {
- lxc-attach -n $1 -- bash /tmp/hive/testutils/metastore/metastore-upgrade-test.sh --db $1
+ lxc-attach -n $1 -- bash hive/testutils/metastore/metastore-upgrade-test.sh --db $1
}
# Install LXC packages if needed
@@ -142,4 +153,6 @@ do
echo "Running metastore upgrade tests for $name..."
run_tests $name
log "$(lxc_print_metastore_log $name)"
+
+ lxc_stop $name
done