You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2015/05/28 09:01:55 UTC
incubator-lens git commit: LENS-514: Addition of an identifier in
every log line for segregating logs
Repository: incubator-lens
Updated Branches:
refs/heads/master 2e90c29c4 -> 4498c4488
LENS-514: Addition of an identifier in every log line for segregating logs
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/4498c448
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/4498c448
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/4498c448
Branch: refs/heads/master
Commit: 4498c4488021afce6ddaf9cdd5e16fca27b2d708
Parents: 2e90c29
Author: Himanshu Gahlaut <hi...@apache.org>
Authored: Thu May 28 12:31:50 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu May 28 12:31:50 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/api/query/QueryHandle.java | 11 +++-
.../lens/api/query/QueryPrepareHandle.java | 4 ++
.../apache/lens/api/response/LensResponse.java | 14 +++--
.../apache/lens/driver/hive/TestHiveDriver.java | 3 +-
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 32 ++++++++---
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 8 +--
.../server/api/query/ExplainQueryContext.java | 11 +++-
.../server/api/query/PreparedQueryContext.java | 6 +-
.../lens/server/api/query/QueryContext.java | 12 ++--
.../server/api/query/QueryExecutionService.java | 8 ++-
.../server/model/LogSegregationContext.java | 35 ++++++++++++
.../MappedDiagnosticLogSegregationContext.java | 36 ++++++++++++
.../lens/server/AuthenticationFilter.java | 3 -
.../org/apache/lens/server/LensApplication.java | 4 ++
.../server/LensRequestContextInitFilter.java | 59 ++++++++++++++++++++
.../java/org/apache/lens/server/LensServer.java | 10 ++--
.../org/apache/lens/server/LensServices.java | 14 ++++-
.../org/apache/lens/server/query/QueryApp.java | 2 +
.../lens/server/query/QueryEndNotifier.java | 14 +++--
.../server/query/QueryExecutionServiceImpl.java | 36 +++++++++---
.../lens/server/query/QueryServiceResource.java | 16 ++++--
.../lens/server/query/ResultFormatter.java | 9 ++-
.../stats/store/log/LogStatisticsStore.java | 7 ++-
.../store/log/StatisticsLogFileScannerTask.java | 12 ++++
.../store/log/StatisticsLogRollupHandler.java | 10 +++-
.../org/apache/lens/server/LensJerseyTest.java | 6 +-
.../lens/server/TestStartupOnMetastoreDown.java | 7 ++-
.../server/query/QueryAPIErrorResponseTest.java | 5 +-
.../stats/TestStatisticsLogFileScannerTask.java | 6 +-
lens-server/src/test/resources/log4j.properties | 4 +-
tools/conf-pseudo-distr/server/log4j.properties | 10 ++--
tools/conf/server/log4j.properties | 10 ++--
32 files changed, 350 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java
index 7b615e6..9f20184 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java
@@ -27,6 +27,8 @@ import java.util.UUID;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.commons.lang.StringUtils;
+
import lombok.*;
/**
@@ -74,6 +76,13 @@ public class QueryHandle extends QuerySubmitResult implements Serializable {
return new QueryHandle(UUID.fromString(handle));
}
+ public String getHandleIdString() {
+ if (handleId == null) {
+ return StringUtils.EMPTY;
+ }
+ return handleId.toString();
+ }
+
/*
* (non-Javadoc)
*
@@ -81,6 +90,6 @@ public class QueryHandle extends QuerySubmitResult implements Serializable {
*/
@Override
public String toString() {
- return handleId.toString();
+ return getHandleIdString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java
index efa04f1..756287e 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java
@@ -75,6 +75,10 @@ public class QueryPrepareHandle extends QuerySubmitResult {
*/
@Override
public String toString() {
+ return getQueryHandleString();
+ }
+
+ public String getQueryHandleString() {
return prepareHandleId.toString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java b/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java
index f6c3593..818ae40 100644
--- a/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java
+++ b/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java
@@ -18,11 +18,15 @@
*/
package org.apache.lens.api.response;
+import static com.google.common.base.Preconditions.checkArgument;
+
import javax.ws.rs.core.Response.Status;
import javax.xml.bind.annotation.*;
import org.apache.lens.api.query.QuerySubmitResult;
+import org.apache.commons.lang.StringUtils;
+
import lombok.*;
/**
@@ -78,12 +82,12 @@ public class LensResponse<DATA, PAYLOAD> {
private LensResponse(final String apiVersion, final String id, final DATA data,
final LensErrorTO lensErrorTO, @NonNull final Status httpStatusCode) {
- /* The checks commented below should be enabled in future, once story of apiVersion and id to be used for log
- tracing is clear. Right now there could be REST APIs throwing LensException without initializing apiVersion
- and id.
+ /* The check commented below should be enabled in future, once story of apiVersion is clear. Right now there could
+ be REST APIs throwing LensException without initializing apiVersion
+
+ checkArgument(StringUtils.isNotBlank(apiVersion)); */
- checkArgument(StringUtils.isNotBlank(apiVersion));
- checkArgument(StringUtils.isNotBlank(id)); */
+ checkArgument(StringUtils.isNotBlank(id));
this.apiVersion = apiVersion;
this.id = id;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 71fbc74..b21258a 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -160,7 +160,8 @@ public class TestHiveDriver {
}
protected ExplainQueryContext createExplainContext(final String query, Configuration conf) {
- ExplainQueryContext ectx = new ExplainQueryContext(query, "testuser", null, conf, drivers);
+ ExplainQueryContext ectx = new ExplainQueryContext(UUID.randomUUID().toString(), query, "testuser", null, conf,
+ drivers);
ectx.setLensSessionIdentifier(sessionid);
return ectx;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 87e4638..70deaa4 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -47,6 +47,8 @@ import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.QueryRewriter;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -56,6 +58,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.log4j.Logger;
import lombok.Getter;
+import lombok.NonNull;
import lombok.Setter;
/**
@@ -89,6 +92,8 @@ public class JDBCDriver implements LensDriver {
/** Estimate connection provider */
private ConnectionProvider estimateConnectionProvider;
+ private LogSegregationContext logSegregationContext;
+
/**
* Data related to a query submitted to JDBCDriver.
*/
@@ -142,12 +147,15 @@ public class JDBCDriver implements LensDriver {
@Setter
private long endTime;
+ private final LogSegregationContext logSegregationContext;
+
/**
* Instantiates a new jdbc query context.
*
* @param context the context
*/
- public JdbcQueryContext(QueryContext context) {
+ public JdbcQueryContext(QueryContext context, @NonNull final LogSegregationContext logSegregationContext) {
+ this.logSegregationContext = logSegregationContext;
this.lensContext = context;
}
@@ -182,6 +190,10 @@ public class JDBCDriver implements LensDriver {
}
isClosed = true;
}
+
+ public String getQueryHandleString() {
+ return this.lensContext.getQueryHandleString();
+ }
}
/**
@@ -261,14 +273,16 @@ public class JDBCDriver implements LensDriver {
/** The query context. */
private final JdbcQueryContext queryContext;
+ private final LogSegregationContext logSegregationContext;
/**
* Instantiates a new query callable.
*
* @param queryContext the query context
*/
- public QueryCallable(JdbcQueryContext queryContext) {
+ public QueryCallable(JdbcQueryContext queryContext, @NonNull LogSegregationContext logSegregationContext) {
this.queryContext = queryContext;
+ this.logSegregationContext = logSegregationContext;
queryContext.setStartTime(System.currentTimeMillis());
}
@@ -279,6 +293,9 @@ public class JDBCDriver implements LensDriver {
*/
@Override
public QueryResult call() {
+
+ logSegregationContext.set(this.queryContext.getQueryHandleString());
+
Statement stmt = null;
Connection conn = null;
QueryResult result = new QueryResult();
@@ -432,6 +449,7 @@ public class JDBCDriver implements LensDriver {
LOG.error("Error initializing connection provider: " + e.getMessage(), e);
throw new LensException(e);
}
+ this.logSegregationContext = new MappedDiagnosticLogSegregationContext();
}
/**
@@ -494,8 +512,6 @@ public class JDBCDriver implements LensDriver {
/**
* Rewrite query.
*
- * @param query the query
- * @param conf the conf
* @return the string
* @throws LensException the lens exception
*/
@@ -850,10 +866,10 @@ public class JDBCDriver implements LensDriver {
*/
private QueryResult executeInternal(QueryContext context, String rewrittenQuery) throws LensException {
- JdbcQueryContext queryContext = new JdbcQueryContext(context);
+ JdbcQueryContext queryContext = new JdbcQueryContext(context, logSegregationContext);
queryContext.setPrepared(false);
queryContext.setRewrittenQuery(rewrittenQuery);
- QueryResult result = new QueryCallable(queryContext).call();
+ QueryResult result = new QueryCallable(queryContext, logSegregationContext).call();
return result;
// LOG.info("Execute " + context.getQueryHandle());
}
@@ -870,10 +886,10 @@ public class JDBCDriver implements LensDriver {
// Always use the driver rewritten query not user query. Since the
// conf we are passing here is query context conf, we need to add jdbc xml in resource path
String rewrittenQuery = rewriteQuery(context);
- JdbcQueryContext jdbcCtx = new JdbcQueryContext(context);
+ JdbcQueryContext jdbcCtx = new JdbcQueryContext(context, logSegregationContext);
jdbcCtx.setRewrittenQuery(rewrittenQuery);
try {
- Future<QueryResult> future = asyncQueryPool.submit(new QueryCallable(jdbcCtx));
+ Future<QueryResult> future = asyncQueryPool.submit(new QueryCallable(jdbcCtx, logSegregationContext));
jdbcCtx.setResultFuture(future);
} catch (RejectedExecutionException e) {
LOG.error("Query execution rejected: " + context.getQueryHandle() + " reason:" + e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index 4f03f23..2e3dcc7 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -24,10 +24,7 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
+import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -131,7 +128,8 @@ public class TestJdbcDriver {
}
protected ExplainQueryContext createExplainContext(final String query, Configuration conf) {
- ExplainQueryContext ectx = new ExplainQueryContext(query, "testuser", null, conf, drivers);
+ ExplainQueryContext ectx = new ExplainQueryContext(UUID.randomUUID().toString(), query, "testuser", null, conf,
+ drivers);
return ectx;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
index da44955..c7b9887 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
@@ -31,14 +31,21 @@ import org.apache.hadoop.conf.Configuration;
public class ExplainQueryContext extends AbstractQueryContext {
private static final long serialVersionUID = 1L;
+ private final String requestId;
/**
* Constructor. Only needs user query and conf.
*
* @param query
* @param qconf
*/
- public ExplainQueryContext(String query, final String user, LensConf conf, Configuration qconf,
- Collection<LensDriver> drivers) {
+ public ExplainQueryContext(final String requestId, String query, final String user, LensConf conf,
+ Configuration qconf, Collection<LensDriver> drivers) {
super(query, user, conf, qconf, drivers, true);
+ this.requestId = requestId;
+ }
+
+ @Override
+ public String getLogHandle() {
+ return this.requestId;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
index 1ce89ac..b6f669b 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
@@ -153,12 +153,16 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye
lensConf);
}
+ public String getQueryHandleString() {
+ return prepareHandle.getQueryHandleString();
+ }
+
/**
* Get prepared query handle string
* @return
*/
@Override
public String getLogHandle() {
- return prepareHandle.getPrepareHandleId().toString();
+ return getQueryHandleString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 169ac8d..3e0c26c 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -368,12 +368,12 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
public boolean getCompressOutput() {
return conf.getBoolean(LensConfConstants.QUERY_OUTPUT_ENABLE_COMPRESSION,
- LensConfConstants.DEFAULT_OUTPUT_ENABLE_COMPRESSION);
+ LensConfConstants.DEFAULT_OUTPUT_ENABLE_COMPRESSION);
}
public long getMaxResultSplitRows() {
return conf.getLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS,
- LensConfConstants.DEFAULT_RESULT_SPLIT_MULTIPLE_MAX_ROWS);
+ LensConfConstants.DEFAULT_RESULT_SPLIT_MULTIPLE_MAX_ROWS);
}
/**
@@ -383,7 +383,7 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
*/
public boolean splitResultIntoMultipleFiles() {
return conf.getBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE,
- LensConfConstants.DEFAULT_RESULT_SPLIT_INTO_MULTIPLE);
+ LensConfConstants.DEFAULT_RESULT_SPLIT_INTO_MULTIPLE);
}
public String getClusterUser() {
@@ -397,6 +397,10 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
*/
@Override
public String getLogHandle() {
- return queryHandle.getHandleId().toString();
+ return getQueryHandleString();
+ }
+
+ public String getQueryHandleString() {
+ return queryHandle.getHandleIdString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
index 8b6d131..3c0f787 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
@@ -40,6 +40,7 @@ public interface QueryExecutionService {
/**
* Estimate the cost of given query.
*
+ * @param requestId the request Id of request used to start estimate operation
* @param sessionHandle the session handle
* @param query The query should be in HiveQL(SQL like)
* @param conf The query configuration
@@ -48,18 +49,21 @@ public interface QueryExecutionService {
*
* @throws LensException thrown in case of failure
*/
- QueryCost estimate(LensSessionHandle sessionHandle, String query, LensConf conf) throws LensException;
+ QueryCost estimate(final String requestId, LensSessionHandle sessionHandle, String query, LensConf conf)
+ throws LensException;
/**
* Explain the given query.
*
+ * @param requestId the request Id of request used to start explain operation
* @param sessionHandle the session handle
* @param query The query should be in HiveQL(SQL like)
* @param conf The query configuration
* @return The query plan;
* @throws LensException the lens exception
*/
- QueryPlan explain(LensSessionHandle sessionHandle, String query, LensConf conf) throws LensException;
+ QueryPlan explain(final String requestId, LensSessionHandle sessionHandle, String query, LensConf conf)
+ throws LensException;
/**
* Prepare the query.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java b/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java
new file mode 100644
index 0000000..8691a05
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.model;
+
+public interface LogSegregationContext {
+
+ /**
+ * Sets an id to be used by current thread in every log line for log segregation
+ *
+ * @param id the id to be added to every log line of current thread
+ */
+ void set(final String id);
+
+ /**
+ *
+ * @return the id being used by the current thread for log segregation
+ */
+ String get();
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java b/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
new file mode 100644
index 0000000..989cb9d
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.model;
+
+import org.slf4j.MDC;
+
+public class MappedDiagnosticLogSegregationContext implements LogSegregationContext {
+
+ private static final String LOG_SEGREGATION_ID = "logSegregationId";
+
+ @Override
+ public void set(String id) {
+ MDC.put(LOG_SEGREGATION_ID, id);
+ }
+
+ @Override
+ public String get() {
+ return MDC.get(LOG_SEGREGATION_ID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java b/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
index b64d822..e39c7a2 100644
--- a/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
+++ b/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
@@ -19,7 +19,6 @@
package org.apache.lens.server;
import java.io.IOException;
-import java.util.UUID;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
@@ -45,9 +44,7 @@ public class AuthenticationFilter implements ContainerRequestFilter {
public void filter(ContainerRequestContext requestContext) throws IOException {
final SecurityContext securityContext = requestContext.getSecurityContext();
- String requestId = UUID.randomUUID().toString();
String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : null;
- requestContext.getHeaders().add("requestId", requestId);
LOG.info("Request from user: " + user + ", path=" + requestContext.getUriInfo().getPath());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
index cb452e8..a3079cc 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
@@ -73,6 +73,10 @@ public class LensApplication extends Application {
classes.add(wsFilterClass);
LOG.info("Added filter " + wsFilterClass);
}
+
+ LOG.debug("LensRequestContextInitFilter added...");
+ classes.add(LensRequestContextInitFilter.class);
+
return classes;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java b/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java
new file mode 100644
index 0000000..c9658fc
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import javax.annotation.Priority;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * LensRequestContextInitFilter is expected to be called before all other request filters.
+ * Priority value of 1 is to ensure the same.
+ *
+ */
+@Slf4j
+@Priority(1)
+public class LensRequestContextInitFilter implements ContainerRequestFilter {
+
+ private static final String REQUEST_ID = "requestId";
+
+ @Override
+ public void filter(ContainerRequestContext requestContext) throws IOException {
+
+ log.debug("Entering {}", getClass().getName());
+
+ /* Create a unique identifier for request */
+ String uniqueRequesId = UUID.randomUUID().toString();
+
+ /* Add request id for appearing in every log line */
+ new MappedDiagnosticLogSegregationContext().set(uniqueRequesId);
+
+ /* Add request id to headers */
+ requestContext.getHeaders().add(REQUEST_ID, uniqueRequesId);
+
+ log.debug("Leaving {}", getClass().getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/LensServer.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServer.java b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
index c6d7ea1..e200bdf 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServer.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
@@ -20,10 +20,7 @@ package org.apache.lens.server;
import java.io.IOException;
import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
+import java.util.*;
import java.util.logging.Logger;
import javax.ws.rs.core.UriBuilder;
@@ -33,6 +30,7 @@ import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.error.LensExceptionMapper;
import org.apache.lens.server.metrics.MetricsServiceImpl;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.ui.UIApp;
import org.apache.commons.logging.Log;
@@ -113,6 +111,7 @@ public class LensServer {
}
private ResourceConfig getApp() {
+
ResourceConfig app = ResourceConfig.forApplicationClass(LensApplication.class);
app.register(new LoggingFilter(Logger.getLogger(LensServer.class.getName() + ".request"), true));
app.register(LensExceptionMapper.class);
@@ -185,6 +184,9 @@ public class LensServer {
*/
public static void main(String[] args) throws Exception {
+ final String runId = UUID.randomUUID().toString();
+ new MappedDiagnosticLogSegregationContext().set(runId);
+
printStartupMessage();
try {
final LensServer thisServer = LensServer.createLensServer(LensServerConf.getHiveConf());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/LensServices.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index f769619..ab197ab 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -33,6 +33,8 @@ import org.apache.lens.server.api.ServiceProvider;
import org.apache.lens.server.api.events.LensEventService;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.metrics.MetricsServiceImpl;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.session.LensSessionImpl;
import org.apache.lens.server.stats.StatisticsService;
import org.apache.lens.server.user.UserConfigLoaderFactory;
@@ -49,6 +51,7 @@ import org.apache.hive.service.Service;
import org.apache.hive.service.cli.CLIService;
import lombok.Getter;
+import lombok.NonNull;
import lombok.Setter;
@@ -68,7 +71,8 @@ public class LensServices extends CompositeService implements ServiceProvider {
private static final String FS_IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
/** The instance. */
- private static LensServices instance = new LensServices(LENS_SERVICES_NAME);
+ private static LensServices instance = new LensServices(LENS_SERVICES_NAME,
+ new MappedDiagnosticLogSegregationContext());
/** The conf. */
private HiveConf conf;
@@ -108,6 +112,9 @@ public class LensServices extends CompositeService implements ServiceProvider {
@Getter
private ErrorCollection errorCollection;
+ @Getter
+ private final LogSegregationContext logSegregationContext;
+
/**
* The Enum SERVICE_MODE.
*/
@@ -129,8 +136,9 @@ public class LensServices extends CompositeService implements ServiceProvider {
*
* @param name the name
*/
- public LensServices(String name) {
+ public LensServices(String name, @NonNull final LogSegregationContext logSegregationContext) {
super(name);
+ this.logSegregationContext = logSegregationContext;
}
// This is only for test, to simulate a restart of the server
@@ -244,6 +252,8 @@ public class LensServices extends CompositeService implements ServiceProvider {
@Override
public void run() {
try {
+ final String runId = UUID.randomUUID().toString();
+ logSegregationContext.set(runId);
persistLensServiceState();
LOG.info("SnapShot of Lens Services created");
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java
index 0d23726..9db034a 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java
@@ -25,6 +25,7 @@ import javax.ws.rs.ApplicationPath;
import javax.ws.rs.core.Application;
import org.apache.lens.server.LensApplicationListener;
+import org.apache.lens.server.LensRequestContextInitFilter;
import org.glassfish.jersey.filter.LoggingFilter;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
@@ -40,6 +41,7 @@ public class QueryApp extends Application {
// register root resource
classes.add(QueryServiceResource.class);
classes.add(MultiPartFeature.class);
+ classes.add(LensRequestContextInitFilter.class);
classes.add(LoggingFilter.class);
classes.add(LensApplicationListener.class);
return classes;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
index 1c6f186..df36a21 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
@@ -37,6 +37,7 @@ import org.apache.lens.server.api.events.AsyncEventListener;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.QueryEnded;
+import org.apache.lens.server.model.LogSegregationContext;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
@@ -44,6 +45,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import lombok.Data;
+import lombok.NonNull;
/**
* The Class QueryEndNotifier.
@@ -77,23 +79,26 @@ public class QueryEndNotifier extends AsyncEventListener<QueryEnded> {
/** The mail smtp connection timeout. */
private final int mailSmtpConnectionTimeout;
+ private final LogSegregationContext logSegregationContext;
+
/**
* Instantiates a new query end notifier.
*
* @param queryService the query service
* @param hiveConf the hive conf
*/
- public QueryEndNotifier(QueryExecutionServiceImpl queryService, HiveConf hiveConf) {
+ public QueryEndNotifier(QueryExecutionServiceImpl queryService, HiveConf hiveConf,
+ @NonNull final LogSegregationContext logSegregationContext) {
this.queryService = queryService;
this.conf = hiveConf;
from = conf.get(LensConfConstants.MAIL_FROM_ADDRESS);
host = conf.get(LensConfConstants.MAIL_HOST);
port = conf.get(LensConfConstants.MAIL_PORT);
- mailSmtpTimeout = Integer.parseInt(conf.get(LensConfConstants.MAIL_SMTP_TIMEOUT,
- LensConfConstants.MAIL_DEFAULT_SMTP_TIMEOUT));
+ mailSmtpTimeout = Integer.parseInt(
+ conf.get(LensConfConstants.MAIL_SMTP_TIMEOUT, LensConfConstants.MAIL_DEFAULT_SMTP_TIMEOUT));
mailSmtpConnectionTimeout = Integer.parseInt(conf.get(LensConfConstants.MAIL_SMTP_CONNECTIONTIMEOUT,
LensConfConstants.MAIL_DEFAULT_SMTP_CONNECTIONTIMEOUT));
-
+ this.logSegregationContext = logSegregationContext;
}
/*
@@ -112,6 +117,7 @@ public class QueryEndNotifier extends AsyncEventListener<QueryEnded> {
+ ". No email generated");
return;
}
+ this.logSegregationContext.set(queryContext.getQueryHandleString());
boolean whetherMailNotify = Boolean.parseBoolean(queryContext.getConf().get(LensConfConstants.QUERY_MAIL_NOTIFY,
LensConfConstants.WHETHER_MAIL_NOTIFY_DEFAULT));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 7c2da3a..dbc52e9 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -51,6 +51,8 @@ import org.apache.lens.server.api.metrics.MethodMetricsContext;
import org.apache.lens.server.api.metrics.MethodMetricsFactory;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.session.LensSessionImpl;
import org.apache.lens.server.stats.StatisticsService;
import org.apache.lens.server.util.UtilityMethods;
@@ -229,6 +231,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
*/
private ExecutorService estimatePool;
+ private final LogSegregationContext logSegregationContext;
+
/**
* The driver event listener.
*/
@@ -250,8 +254,10 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* @param cliService the cli service
* @throws LensException the lens exception
*/
- public QueryExecutionServiceImpl(CLIService cliService) throws LensException {
+ public QueryExecutionServiceImpl(CLIService cliService)
+ throws LensException {
super(NAME, cliService);
+ this.logSegregationContext = new MappedDiagnosticLogSegregationContext();
}
/**
@@ -280,10 +286,11 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
log.info("Registered query state logger");
}
// Add result formatter
- getEventService().addListenerForType(new ResultFormatter(this), QueryExecuted.class);
+ getEventService().addListenerForType(new ResultFormatter(this, this.logSegregationContext), QueryExecuted.class);
getEventService().addListenerForType(new QueryExecutionStatisticsGenerator(this, getEventService()),
QueryEnded.class);
- getEventService().addListenerForType(new QueryEndNotifier(this, getCliService().getHiveConf()), QueryEnded.class);
+ getEventService().addListenerForType(
+ new QueryEndNotifier(this, getCliService().getHiveConf(), this.logSegregationContext), QueryEnded.class);
log.info("Registered query result formatter");
}
@@ -456,6 +463,10 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public QueryContext getCtx() {
return ctx;
}
+
+ public String getQueryHandleString() {
+ return ctx.getQueryHandleString();
+ }
}
/**
@@ -479,6 +490,10 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
while (!pausedForTest && !stopped && !querySubmitter.isInterrupted()) {
try {
QueryContext ctx = queuedQueries.take();
+
+ /* Setting log segregation id */
+ logSegregationContext.set(ctx.getQueryHandleString());
+
synchronized (ctx) {
if (ctx.getStatus().getStatus().equals(Status.QUEUED)) {
log.info("Launching query:" + ctx.getUserQuery());
@@ -553,6 +568,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
if (stopped || statusPoller.isInterrupted()) {
return;
}
+
+ logSegregationContext.set(ctx.getQueryHandleString());
log.info("Polling status for " + ctx.getQueryHandle());
try {
// session is not required to update status of the query
@@ -762,6 +779,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
FinishedQuery finished = null;
try {
finished = finishedQueries.take();
+ logSegregationContext.set(finished.getQueryHandleString());
} catch (InterruptedException e) {
log.info("QueryPurger has been interrupted, exiting");
return;
@@ -835,6 +853,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
while (!stopped && !prepareQueryPurger.isInterrupted()) {
try {
PreparedQueryContext prepared = preparedQueryQueue.take();
+ logSegregationContext.set(prepared.getQueryHandleString());
destroyPreparedQuery(prepared);
log.info("Purged prepared query: " + prepared.getPrepareHandle());
} catch (LensException e) {
@@ -1005,6 +1024,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
};
+ LOG.debug("starting estimate pool");
ThreadPoolExecutor estimatePool = new ThreadPoolExecutor(minPoolSize, maxPoolSize, keepAlive, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(), threadFactory);
estimatePool.allowCoreThreadTimeOut(true);
@@ -1163,6 +1183,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
@Override
public void run() {
try {
+ logSegregationContext.set(ctx.getLogHandle());
acquire(ctx.getLensSessionIdentifier());
MethodMetricsContext rewriteGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true,
REWRITE_GAUGE);
@@ -2021,13 +2042,13 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* java.lang.String, org.apache.lens.api.LensConf)
*/
@Override
- public QueryCost estimate(LensSessionHandle sessionHandle, String query, LensConf lensConf)
+ public QueryCost estimate(final String requestId, LensSessionHandle sessionHandle, String query, LensConf lensConf)
throws LensException {
try {
log.info("Estimate: " + sessionHandle.toString() + " query:" + query);
acquire(sessionHandle);
Configuration qconf = getLensConf(sessionHandle, lensConf);
- ExplainQueryContext estimateQueryContext = new ExplainQueryContext(query,
+ ExplainQueryContext estimateQueryContext = new ExplainQueryContext(requestId, query,
getSession(sessionHandle).getLoggedInUser(), lensConf, qconf, drivers.values());
estimateQueryContext.setLensSessionIdentifier(sessionHandle.getPublicId().toString());
accept(query, qconf, SubmitOp.ESTIMATE);
@@ -2045,12 +2066,13 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* java.lang.String, org.apache.lens.api.LensConf)
*/
@Override
- public QueryPlan explain(LensSessionHandle sessionHandle, String query, LensConf lensConf) throws LensException {
+ public QueryPlan explain(final String requestId, LensSessionHandle sessionHandle, String query, LensConf lensConf)
+ throws LensException {
try {
log.info("Explain: " + sessionHandle.toString() + " query:" + query);
acquire(sessionHandle);
Configuration qconf = getLensConf(sessionHandle, lensConf);
- ExplainQueryContext explainQueryContext = new ExplainQueryContext(query,
+ ExplainQueryContext explainQueryContext = new ExplainQueryContext(requestId, query,
getSession(sessionHandle).getLoggedInUser(), lensConf, qconf, drivers.values());
explainQueryContext.setLensSessionIdentifier(sessionHandle.getPublicId().toString());
accept(query, qconf, SubmitOp.EXPLAIN);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
index 34e14b6..badde8c 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
@@ -40,6 +40,7 @@ import org.apache.lens.server.api.annotations.MultiPurposeResource;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.QueryExecutionService;
import org.apache.lens.server.error.UnSupportedQuerySubmitOpException;
+import org.apache.lens.server.model.LogSegregationContext;
import org.apache.commons.lang.StringUtils;
@@ -61,6 +62,8 @@ public class QueryServiceResource {
private final ErrorCollection errorCollection;
+ private final LogSegregationContext logSegregationContext;
+
/**
* Check session id.
*
@@ -123,6 +126,7 @@ public class QueryServiceResource {
public QueryServiceResource() throws LensException {
queryServer = (QueryExecutionService) LensServices.get().getService("query");
errorCollection = LensServices.get().getErrorCollection();
+ logSegregationContext = LensServices.get().getLogSegregationContext();
}
QueryExecutionService getQueryServer() {
@@ -201,8 +205,9 @@ public class QueryServiceResource {
@FormDataParam("conf") LensConf conf, @DefaultValue("30000") @FormDataParam("timeoutmillis") Long timeoutmillis,
@DefaultValue("") @FormDataParam("queryName") String queryName) throws LensException {
- try {
+ final String requestId = this.logSegregationContext.get();
+ try {
validateSessionId(sessionid);
SubmitOp sop = checkAndGetQuerySubmitOperation(operation);
validateQuery(query);
@@ -210,13 +215,13 @@ public class QueryServiceResource {
QuerySubmitResult result;
switch (sop) {
case ESTIMATE:
- result = queryServer.estimate(sessionid, query, conf);
+ result = queryServer.estimate(requestId, sessionid, query, conf);
break;
case EXECUTE:
result = queryServer.executeAsync(sessionid, query, conf, queryName);
break;
case EXPLAIN:
- result = queryServer.explain(sessionid, query, conf);
+ result = queryServer.explain(requestId, sessionid, query, conf);
break;
case EXECUTE_WITH_TIMEOUT:
result = queryServer.execute(sessionid, query, timeoutmillis, conf, queryName);
@@ -224,9 +229,10 @@ public class QueryServiceResource {
default:
throw new UnSupportedQuerySubmitOpException();
}
- return LensResponse.composedOf(null, null, result);
+
+ return LensResponse.composedOf(null, requestId, result);
} catch (LensException e) {
- e.buildLensErrorResponse(errorCollection, null, null);
+ e.buildLensErrorResponse(errorCollection, null, requestId);
throw e;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
index 7f5ed65..de7e120 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
@@ -28,6 +28,7 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.AsyncEventListener;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.model.LogSegregationContext;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -35,6 +36,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.ReflectionUtils;
+import lombok.NonNull;
+
/**
* The Class ResultFormatter.
*/
@@ -46,13 +49,16 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> {
/** The query service. */
QueryExecutionServiceImpl queryService;
+ private final LogSegregationContext logSegregationContext;
+
/**
* Instantiates a new result formatter.
*
* @param queryService the query service
*/
- public ResultFormatter(QueryExecutionServiceImpl queryService) {
+ public ResultFormatter(QueryExecutionServiceImpl queryService, @NonNull LogSegregationContext logSegregationContext) {
this.queryService = queryService;
+ this.logSegregationContext = logSegregationContext;
}
/*
@@ -73,6 +79,7 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> {
private void formatOutput(QueryExecuted event) {
QueryHandle queryHandle = event.getQueryHandle();
QueryContext ctx = queryService.getQueryContext(queryHandle);
+ this.logSegregationContext.set(ctx.getQueryHandleString());
try {
if (!ctx.isPersistent()) {
LOG.info("No result formatting required for query " + queryHandle);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java
index 131a375..81ad260 100644
--- a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java
+++ b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java
@@ -21,6 +21,8 @@ package org.apache.lens.server.stats.store.log;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.events.LensEventService;
import org.apache.lens.server.api.metrics.MetricsService;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.stats.event.LoggableLensStatistics;
import org.apache.lens.server.stats.store.StatisticsStore;
@@ -53,11 +55,14 @@ public class LogStatisticsStore extends StatisticsStore<LoggableLensStatistics>
/** The rollup handler. */
private StatisticsLogRollupHandler rollupHandler;
+ private final LogSegregationContext logSegregationContext;
+
/**
* Instantiates a new log statistics store.
*/
public LogStatisticsStore() {
this.mapper = new ObjectMapper();
+ this.logSegregationContext = new MappedDiagnosticLogSegregationContext();
mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
}
@@ -71,7 +76,7 @@ public class LogStatisticsStore extends StatisticsStore<LoggableLensStatistics>
handler = new StatisticsLogPartitionHandler();
handler.initialize(conf);
LOG.info("Creating new rollup handler");
- rollupHandler = new StatisticsLogRollupHandler();
+ rollupHandler = new StatisticsLogRollupHandler(this.logSegregationContext);
rollupHandler.initialize(conf);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
index 254632b..7e9c74d 100644
--- a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
+++ b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
@@ -23,18 +23,21 @@ import java.io.FilenameFilter;
import java.util.HashMap;
import java.util.Map;
import java.util.TimerTask;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.LensEventService;
import org.apache.lens.server.api.metrics.MetricsService;
+import org.apache.lens.server.model.LogSegregationContext;
import org.apache.log4j.FileAppender;
import org.apache.log4j.Logger;
import org.slf4j.LoggerFactory;
+import lombok.NonNull;
import lombok.Setter;
/**
@@ -58,6 +61,11 @@ public class StatisticsLogFileScannerTask extends TimerTask {
/** The class set. */
private Map<String, String> classSet = new ConcurrentHashMap<String, String>();
+ private final LogSegregationContext logSegregationContext;
+
+ public StatisticsLogFileScannerTask(@NonNull final LogSegregationContext logSegregationContext) {
+ this.logSegregationContext = logSegregationContext;
+ }
/*
* (non-Javadoc)
*
@@ -66,6 +74,10 @@ public class StatisticsLogFileScannerTask extends TimerTask {
@Override
public void run() {
try {
+
+ final String runId = UUID.randomUUID().toString();
+ this.logSegregationContext.set(runId);
+
for (Map.Entry<String, String> entry : scanSet.entrySet()) {
File f = new File(entry.getValue()).getAbsoluteFile();
String fileName = f.getAbsolutePath();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java
index 78e4518..9ff878c 100644
--- a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java
+++ b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java
@@ -22,11 +22,14 @@ import java.util.Timer;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.events.LensEventService;
+import org.apache.lens.server.model.LogSegregationContext;
import org.apache.hadoop.conf.Configuration;
import org.eclipse.jetty.util.ConcurrentHashSet;
+import lombok.NonNull;
+
/**
* Class which handles the log4j rolled file.
*/
@@ -44,13 +47,18 @@ public class StatisticsLogRollupHandler {
/** The scan set. */
private final ConcurrentHashSet<String> scanSet = new ConcurrentHashSet<String>();
+ private final LogSegregationContext logSegregationContext;
+
+ public StatisticsLogRollupHandler(@NonNull final LogSegregationContext logSegregationContext) {
+ this.logSegregationContext = logSegregationContext;
+ }
/**
* Initalize the handler.
*
* @param conf configuration to be used while initialization.
*/
public void initialize(Configuration conf) {
- task = new StatisticsLogFileScannerTask();
+ task = new StatisticsLogFileScannerTask(this.logSegregationContext);
timer = new Timer();
rate = conf.getLong(LensConfConstants.STATS_ROLLUP_SCAN_RATE, LensConfConstants.DEFAULT_STATS_ROLLUP_SCAN_RATE);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
index 7913845..fb3e67b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
@@ -30,6 +30,8 @@ import org.apache.lens.driver.hive.TestRemoteHiveDriver;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.metrics.LensMetricsUtil;
import org.apache.lens.server.api.metrics.MetricsService;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -50,6 +52,8 @@ public abstract class LensJerseyTest extends JerseyTest {
private int port = -1;
+ private final LogSegregationContext logSegregationContext = new MappedDiagnosticLogSegregationContext();
+
protected URI getUri() {
return UriBuilder.fromUri("http://localhost/").port(getTestPort()).build();
}
@@ -195,7 +199,7 @@ public abstract class LensJerseyTest extends JerseyTest {
LensServices.get().stop();
LensMetricsUtil.clearRegistry();
System.out.println("Lens services stopped!");
- LensServices.setInstance(new LensServices(LensServices.LENS_SERVICES_NAME));
+ LensServices.setInstance(new LensServices(LensServices.LENS_SERVICES_NAME, this.logSegregationContext));
LensServices.get().init(conf);
LensServices.get().start();
System.out.println("Lens services restarted!");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java b/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
index decd6ef..3fb123e 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
@@ -18,6 +18,9 @@
*/
package org.apache.lens.server;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -28,9 +31,11 @@ import org.testng.Assert;
public class TestStartupOnMetastoreDown {
private static final Log LOG = LogFactory.getLog(TestStartupOnMetastoreDown.class);
+ private final LogSegregationContext logSegregationContext = new MappedDiagnosticLogSegregationContext();
+
// @Test
public void testServicesStartOnMetastoreDown() throws Exception {
- LensServices services = new LensServices(LensServices.LENS_SERVICES_NAME);
+ LensServices services = new LensServices(LensServices.LENS_SERVICES_NAME, logSegregationContext);
HiveConf hiveConf = new HiveConf();
// Set metastore uri to an invalid location
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
index 5e135be..4c2a7a4 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
@@ -45,6 +45,7 @@ import org.apache.lens.api.response.LensErrorTO;
import org.apache.lens.api.response.LensJAXBContextResolver;
import org.apache.lens.cube.error.ColUnAvailableInTimeRange;
import org.apache.lens.server.LensJerseyTest;
+import org.apache.lens.server.LensRequestContextInitFilter;
import org.apache.lens.server.common.ErrorResponseExpectedData;
import org.apache.lens.server.error.LensExceptionMapper;
import org.apache.lens.server.metastore.MetastoreResource;
@@ -89,8 +90,8 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest {
enable(TestProperties.LOG_TRAFFIC);
enable(TestProperties.DUMP_ENTITY);
- return new ResourceConfig(SessionResource.class, MetastoreResource.class, QueryServiceResource.class,
- MultiPartFeature.class, LensExceptionMapper.class, LensJAXBContextResolver.class);
+ return new ResourceConfig(LensRequestContextInitFilter.class, SessionResource.class, MetastoreResource.class,
+ QueryServiceResource.class, MultiPartFeature.class, LensExceptionMapper.class, LensJAXBContextResolver.class);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java b/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java
index d5e60c4..819c62a 100644
--- a/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java
+++ b/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java
@@ -23,6 +23,8 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lens.server.api.events.LensEventService;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.stats.store.log.PartitionEvent;
import org.apache.lens.server.stats.store.log.StatisticsLogFileScannerTask;
@@ -52,6 +54,8 @@ public class TestStatisticsLogFileScannerTask {
/** The hidden. */
private File hidden;
+ private final LogSegregationContext logSegregationContext = new MappedDiagnosticLogSegregationContext();
+
/**
* Creates the test log file.
*
@@ -90,7 +94,7 @@ public class TestStatisticsLogFileScannerTask {
appender.setName(TestStatisticsLogFileScannerTask.class.getSimpleName());
l.addAppender(appender);
- StatisticsLogFileScannerTask task = new StatisticsLogFileScannerTask();
+ StatisticsLogFileScannerTask task = new StatisticsLogFileScannerTask(this.logSegregationContext);
task.addLogFile(TestStatisticsLogFileScannerTask.class.getName());
LensEventService service = Mockito.mock(LensEventService.class);
final List<PartitionEvent> events = new ArrayList<PartitionEvent>();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/log4j.properties b/lens-server/src/test/resources/log4j.properties
index 65d065f..f3ccea2 100644
--- a/lens-server/src/test/resources/log4j.properties
+++ b/lens-server/src/test/resources/log4j.properties
@@ -31,11 +31,11 @@ log4j.additivity.org.apache.lens.server.stats.TestLogStatisticsStore$MyLoggableL
# Configuration for appenders
log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender
log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout
-log4j.appender.STDOUT.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
+log4j.appender.STDOUT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.STDOUT.Threshold=ERROR
log4j.appender.TEST_LOG_FILE=org.apache.log4j.RollingFileAppender
log4j.appender.TEST_LOG_FILE.File=lens-server/target/test.log
log4j.appender.TEST_LOG_FILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.TEST_LOG_FILE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
+log4j.appender.TEST_LOG_FILE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %X{logSegregationId} [%t] %-5p %c %x - %m%n
log4j.appender.TEST_LOG_FILE.Threshold=ERROR
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/tools/conf-pseudo-distr/server/log4j.properties
----------------------------------------------------------------------
diff --git a/tools/conf-pseudo-distr/server/log4j.properties b/tools/conf-pseudo-distr/server/log4j.properties
index afadc2f..2b04ea5 100644
--- a/tools/conf-pseudo-distr/server/log4j.properties
+++ b/tools/conf-pseudo-distr/server/log4j.properties
@@ -38,12 +38,12 @@ log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.ROOT=org.apache.log4j.RollingFileAppender
log4j.appender.ROOT.File=${lens.log.dir}/lensserver.log
log4j.appender.ROOT.layout=org.apache.log4j.PatternLayout
-log4j.appender.ROOT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.ROOT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.ROOT.MaxFileSize=100000KB
# Keep 20 backup files
@@ -53,7 +53,7 @@ log4j.appender.ROOT.MaxBackupIndex=20
log4j.appender.AUDIT=org.apache.log4j.RollingFileAppender
log4j.appender.AUDIT.File=${lens.log.dir}/lensserver-audit.log
log4j.appender.AUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.AUDIT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.AUDIT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.AUDIT.MaxFileSize=100000KB
# Keep 20 backup files
@@ -62,7 +62,7 @@ log4j.appender.AUDIT.MaxBackupIndex=20
log4j.appender.REQUEST=org.apache.log4j.RollingFileAppender
log4j.appender.REQUEST.File=${lens.log.dir}/lensserver-requests.log
log4j.appender.REQUEST.layout=org.apache.log4j.PatternLayout
-log4j.appender.REQUEST.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.REQUEST.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.REQUEST.MaxFileSize=100000KB
# Keep 20 backup files
@@ -71,7 +71,7 @@ log4j.appender.REQUEST.MaxBackupIndex=20
log4j.appender.STATUS=org.apache.log4j.RollingFileAppender
log4j.appender.STATUS.File=${lens.log.dir}/lensserver-query-status.log
log4j.appender.STATUS.layout=org.apache.log4j.PatternLayout
-log4j.appender.STATUS.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.STATUS.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.STATUS.MaxFileSize=100000KB
# Keep 20 backup files
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/tools/conf/server/log4j.properties
----------------------------------------------------------------------
diff --git a/tools/conf/server/log4j.properties b/tools/conf/server/log4j.properties
index afadc2f..2b04ea5 100644
--- a/tools/conf/server/log4j.properties
+++ b/tools/conf/server/log4j.properties
@@ -38,12 +38,12 @@ log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.ROOT=org.apache.log4j.RollingFileAppender
log4j.appender.ROOT.File=${lens.log.dir}/lensserver.log
log4j.appender.ROOT.layout=org.apache.log4j.PatternLayout
-log4j.appender.ROOT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.ROOT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.ROOT.MaxFileSize=100000KB
# Keep 20 backup files
@@ -53,7 +53,7 @@ log4j.appender.ROOT.MaxBackupIndex=20
log4j.appender.AUDIT=org.apache.log4j.RollingFileAppender
log4j.appender.AUDIT.File=${lens.log.dir}/lensserver-audit.log
log4j.appender.AUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.AUDIT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.AUDIT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.AUDIT.MaxFileSize=100000KB
# Keep 20 backup files
@@ -62,7 +62,7 @@ log4j.appender.AUDIT.MaxBackupIndex=20
log4j.appender.REQUEST=org.apache.log4j.RollingFileAppender
log4j.appender.REQUEST.File=${lens.log.dir}/lensserver-requests.log
log4j.appender.REQUEST.layout=org.apache.log4j.PatternLayout
-log4j.appender.REQUEST.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.REQUEST.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.REQUEST.MaxFileSize=100000KB
# Keep 20 backup files
@@ -71,7 +71,7 @@ log4j.appender.REQUEST.MaxBackupIndex=20
log4j.appender.STATUS=org.apache.log4j.RollingFileAppender
log4j.appender.STATUS.File=${lens.log.dir}/lensserver-query-status.log
log4j.appender.STATUS.layout=org.apache.log4j.PatternLayout
-log4j.appender.STATUS.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.STATUS.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.STATUS.MaxFileSize=100000KB
# Keep 20 backup files